diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h index fcd522641b..dc5f6eb143 100644 --- a/include/crm/pengine/status.h +++ b/include/crm/pengine/status.h @@ -1,495 +1,502 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef PENGINE_STATUS__H # define PENGINE_STATUS__H # include # include # include # include typedef struct node_s pe_node_t; typedef struct node_s node_t; typedef struct pe_action_s action_t; typedef struct pe_action_s pe_action_t; typedef struct resource_s resource_t; typedef struct ticket_s ticket_t; typedef enum no_quorum_policy_e { no_quorum_freeze, no_quorum_stop, no_quorum_ignore, no_quorum_suicide } no_quorum_policy_t; enum node_type { node_ping, node_member, node_remote }; enum pe_restart { pe_restart_restart, pe_restart_ignore }; enum pe_find { pe_find_renamed = 0x001, pe_find_clone = 0x004, pe_find_current = 0x008, pe_find_inactive = 0x010, }; # define pe_flag_have_quorum 0x00000001ULL # define pe_flag_symmetric_cluster 0x00000002ULL # define pe_flag_is_managed_default 0x00000004ULL # define pe_flag_maintenance_mode 0x00000008ULL # define pe_flag_stonith_enabled 0x00000010ULL # define pe_flag_have_stonith_resource 0x00000020ULL # define pe_flag_enable_unfencing 0x00000040ULL # define pe_flag_concurrent_fencing 0x00000080ULL # define pe_flag_stop_rsc_orphans 0x00000100ULL # define pe_flag_stop_action_orphans 0x00000200ULL # define pe_flag_stop_everything 0x00000400ULL # define pe_flag_start_failure_fatal 0x00001000ULL # define pe_flag_remove_after_stop 0x00002000ULL # define pe_flag_startup_probes 0x00010000ULL # define pe_flag_have_status 0x00020000ULL # define pe_flag_have_remote_nodes 0x00040000ULL # define pe_flag_quick_location 0x00100000ULL # define pe_flag_sanitized 0x00200000ULL typedef struct pe_working_set_s { xmlNode *input; crm_time_t *now; /* options extracted from the input */ char *dc_uuid; node_t *dc_node; const char *stonith_action; const char *placement_strategy; unsigned long long flags; int stonith_timeout; int default_resource_stickiness; no_quorum_policy_t no_quorum_policy; GHashTable *config_hash; GHashTable *tickets; GHashTable *singletons; /* Actions for which there can be only one - ie. fence nodeX */ GListPtr nodes; GListPtr resources; GListPtr placement_constraints; GListPtr ordering_constraints; GListPtr colocation_constraints; GListPtr ticket_constraints; GListPtr actions; xmlNode *failed; xmlNode *op_defaults; xmlNode *rsc_defaults; /* stats */ int num_synapse; int max_valid_nodes; int order_id; int action_id; /* final output */ xmlNode *graph; GHashTable *template_rsc_sets; const char *localhost; GHashTable *tags; int blocked_resources; int disabled_resources; } pe_working_set_t; struct node_shared_s { const char *id; const char *uname; /* @TODO convert these flags (and the ones at the end) into a bitfield */ gboolean online; gboolean standby; gboolean standby_onfail; gboolean pending; gboolean unclean; gboolean unseen; gboolean shutdown; gboolean expected_up; gboolean is_dc; int num_resources; GListPtr running_rsc; /* resource_t* */ GListPtr allocated_rsc; /* resource_t* */ resource_t *remote_rsc; GHashTable *attrs; /* char* => char* */ enum node_type type; GHashTable *utilization; /*! cache of calculated rsc digests for this node. */ GHashTable *digest_cache; gboolean maintenance; gboolean rsc_discovery_enabled; gboolean remote_requires_reset; gboolean remote_was_fenced; gboolean remote_maintenance; /* what the remote-rsc is thinking */ gboolean unpacked; }; struct node_s { int weight; gboolean fixed; int count; struct node_shared_s *details; int rsc_discover_mode; }; # include # define pe_rsc_orphan 0x00000001ULL # define pe_rsc_managed 0x00000002ULL # define pe_rsc_block 0x00000004ULL /* Further operations are prohibited due to failure policy */ # define pe_rsc_orphan_container_filler 0x00000008ULL # define pe_rsc_notify 0x00000010ULL # define pe_rsc_unique 0x00000020ULL # define pe_rsc_fence_device 0x00000040ULL # define pe_rsc_provisional 0x00000100ULL # define pe_rsc_allocating 0x00000200ULL # define pe_rsc_merging 0x00000400ULL # define pe_rsc_munging 0x00000800ULL # define pe_rsc_try_reload 0x00001000ULL # define pe_rsc_reload 0x00002000ULL # define pe_rsc_allow_remote_remotes 0x00004000ULL # define pe_rsc_failed 0x00010000ULL # define pe_rsc_shutdown 0x00020000ULL # define pe_rsc_runnable 0x00040000ULL # define pe_rsc_start_pending 0x00080000ULL # define pe_rsc_starting 0x00100000ULL # define pe_rsc_stopping 0x00200000ULL # define pe_rsc_migrating 0x00400000ULL # define pe_rsc_allow_migrate 0x00800000ULL # define pe_rsc_failure_ignored 0x01000000ULL # define pe_rsc_unexpectedly_running 0x02000000ULL # define pe_rsc_maintenance 0x04000000ULL # define pe_rsc_needs_quorum 0x10000000ULL # define pe_rsc_needs_fencing 0x20000000ULL # define pe_rsc_needs_unfencing 0x40000000ULL # define pe_rsc_have_unfencing 0x80000000ULL /* obsolete (not set or used by cluster) */ enum pe_graph_flags { pe_graph_none = 0x00000, pe_graph_updated_first = 0x00001, pe_graph_updated_then = 0x00002, pe_graph_disable = 0x00004, }; /* *INDENT-OFF* */ enum pe_action_flags { pe_action_pseudo = 0x00001, pe_action_runnable = 0x00002, pe_action_optional = 0x00004, pe_action_print_always = 0x00008, pe_action_have_node_attrs = 0x00010, pe_action_failure_is_fatal = 0x00020, /* no longer used, here for API compatibility */ pe_action_implied_by_stonith = 0x00040, pe_action_migrate_runnable = 0x00080, pe_action_dumped = 0x00100, pe_action_processed = 0x00200, pe_action_clear = 0x00400, pe_action_dangle = 0x00800, /* This action requires one or more of its dependencies to be runnable. * We use this to clear the runnable flag before checking dependencies. */ pe_action_requires_any = 0x01000, pe_action_reschedule = 0x02000, pe_action_tracking = 0x04000, }; /* *INDENT-ON* */ struct resource_s { char *id; char *clone_name; xmlNode *xml; xmlNode *orig_xml; xmlNode *ops_xml; resource_t *parent; void *variant_opaque; enum pe_obj_types variant; resource_object_functions_t *fns; resource_alloc_functions_t *cmds; enum rsc_recovery_type recovery_type; enum pe_restart restart_type; int priority; int stickiness; int sort_index; int failure_timeout; int effective_priority; int migration_threshold; gboolean is_remote_node; unsigned long long flags; GListPtr rsc_cons_lhs; /* rsc_colocation_t* */ GListPtr rsc_cons; /* rsc_colocation_t* */ GListPtr rsc_location; /* rsc_to_node_t* */ GListPtr actions; /* action_t* */ GListPtr rsc_tickets; /* rsc_ticket* */ node_t *allocated_to; GListPtr running_on; /* node_t* */ GHashTable *known_on; /* node_t* */ GHashTable *allowed_nodes; /* node_t* */ enum rsc_role_e role; enum rsc_role_e next_role; GHashTable *meta; GHashTable *parameters; GHashTable *utilization; GListPtr children; /* resource_t* */ GListPtr dangling_migrations; /* node_t* */ node_t *partial_migration_target; node_t *partial_migration_source; resource_t *container; GListPtr fillers; char *pending_task; const char *isolation_wrapper; gboolean exclusive_discover; int remote_reconnect_interval; pe_working_set_t *cluster; xmlNode *versioned_parameters; }; // Used as action->action_details if action->rsc is not NULL typedef struct pe_rsc_action_details_s { xmlNode *versioned_parameters; xmlNode *versioned_meta; } pe_rsc_action_details_t; struct pe_action_s { int id; int priority; resource_t *rsc; node_t *node; xmlNode *op_entry; char *task; char *uuid; char *reason; char *cancel_task; enum pe_action_flags flags; enum rsc_start_requirement needs; enum action_fail_response on_fail; enum rsc_role_e fail_role; action_t *pre_notify; action_t *pre_notified; action_t *post_notify; action_t *post_notified; int seen_count; GHashTable *meta; GHashTable *extra; /* * These two varables are associated with the constraint logic * that involves first having one or more actions runnable before * then allowing this action to execute. * * These varables are used with features such as 'clone-min' which * requires at minimum X number of cloned instances to be running * before an order dependency can run. Another option that uses * this is 'require-all=false' in ordering constrants. This option * says "only require one instance of a resource to start before * allowing dependencies to start" -- basically, require-all=false is * the same as clone-min=1. */ /* current number of known runnable actions in the before list. */ int runnable_before; /* the number of "before" runnable actions required for this action * to be considered runnable */ int required_runnable_before; GListPtr actions_before; /* action_wrapper_t* */ GListPtr actions_after; /* action_wrapper_t* */ /* Some of the above fields could be moved to the details, * except for API backward compatibility. */ void *action_details; // varies by type of action }; struct ticket_s { char *id; gboolean granted; time_t last_granted; gboolean standby; GHashTable *state; }; typedef struct tag_s { char *id; GListPtr refs; } tag_t; enum pe_link_state { pe_link_not_dumped, pe_link_dumped, pe_link_dup, }; + +enum rsc_discover_e { + discover_always = 0, + discover_never, + discover_exclusive, +}; + /* *INDENT-OFF* */ enum pe_ordering { pe_order_none = 0x0, /* deleted */ pe_order_optional = 0x1, /* pure ordering, nothing implied */ pe_order_apply_first_non_migratable = 0x2, /* Only apply this constraint's ordering if first is not migratable. */ pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */ pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */ pe_order_implies_first_master = 0x40, /* Imply 'first' is required when 'then' is required and then's rsc holds Master role. */ /* first requires then to be both runnable and migrate runnable. */ pe_order_implies_first_migratable = 0x80, pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */ pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */ pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX', * ensure instances of 'then' on 'nodeX' are too. * Only really useful if 'then' is a clone and 'first' is not */ pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */ pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */ pe_order_serialize_only = 0x4000, /* serialize */ pe_order_same_node = 0x8000, /* applies only if 'first' and 'then' are on same node */ pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not mandatory */ pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not mandatory */ pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */ pe_order_load = 0x200000, /* Only relevant if... */ pe_order_one_or_more = 0x400000, /* 'then' is runnable only if one or more of its dependencies are too */ pe_order_anti_colocation = 0x800000, pe_order_preserve = 0x1000000, /* Hack for breaking user ordering constraints with container resources */ pe_order_trace = 0x4000000, /* test marker */ }; /* *INDENT-ON* */ typedef struct action_wrapper_s action_wrapper_t; struct action_wrapper_s { enum pe_ordering type; enum pe_link_state state; action_t *action; }; const char *rsc_printable_id(resource_t *rsc); gboolean cluster_status(pe_working_set_t * data_set); void set_working_set_defaults(pe_working_set_t * data_set); void cleanup_calculations(pe_working_set_t * data_set); resource_t *pe_find_resource(GListPtr rsc_list, const char *id_rh); node_t *pe_find_node(GListPtr node_list, const char *uname); node_t *pe_find_node_id(GListPtr node_list, const char *id); node_t *pe_find_node_any(GListPtr node_list, const char *id, const char *uname); GListPtr find_operations(const char *rsc, const char *node, gboolean active_filter, pe_working_set_t * data_set); pe_rsc_action_details_t *pe_rsc_action_details(pe_action_t *action); /*! * \brief Check whether a resource is any clone type * * \param[in] rsc Resource to check * * \return TRUE if resource is clone, FALSE otherwise */ static inline bool pe_rsc_is_clone(resource_t *rsc) { return rsc && ((rsc->variant == pe_clone) || (rsc->variant == pe_master)); } /*! * \brief Check whether a resource is a globally unique clone * * \param[in] rsc Resource to check * * \return TRUE if resource is unique clone, FALSE otherwise */ static inline bool pe_rsc_is_unique_clone(resource_t *rsc) { return pe_rsc_is_clone(rsc) && is_set(rsc->flags, pe_rsc_unique); } /*! * \brief Check whether a resource is an anonymous clone * * \param[in] rsc Resource to check * * \return TRUE if resource is anonymous clone, FALSE otherwise */ static inline bool pe_rsc_is_anon_clone(resource_t *rsc) { return pe_rsc_is_clone(rsc) && is_not_set(rsc->flags, pe_rsc_unique); } #endif diff --git a/lib/pengine/container.c b/lib/pengine/container.c index 5831b58e3b..a550ff33a3 100644 --- a/lib/pengine/container.c +++ b/lib/pengine/container.c @@ -1,1108 +1,1111 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #define VARIANT_CONTAINER 1 #include "./variant.h" void tuple_free(container_grouping_t *tuple); static char * next_ip(const char *last_ip) { unsigned int oct1 = 0; unsigned int oct2 = 0; unsigned int oct3 = 0; unsigned int oct4 = 0; int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4); if (rc != 4) { /*@ TODO check for IPv6 */ return NULL; } else if (oct3 > 253) { return NULL; } else if (oct4 > 253) { ++oct3; oct4 = 1; } else { ++oct4; } return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4); } static int allocate_ip(container_variant_data_t *data, container_grouping_t *tuple, char *buffer, int max) { if(data->ip_range_start == NULL) { return 0; } else if(data->ip_last) { tuple->ipaddr = next_ip(data->ip_last); } else { tuple->ipaddr = strdup(data->ip_range_start); } data->ip_last = tuple->ipaddr; #if 0 return snprintf(buffer, max, " --add-host=%s-%d:%s --link %s-docker-%d:%s-link-%d", data->prefix, tuple->offset, tuple->ipaddr, data->prefix, tuple->offset, data->prefix, tuple->offset); #else return snprintf(buffer, max, " --add-host=%s-%d:%s", data->prefix, tuple->offset, tuple->ipaddr); #endif } static xmlNode * create_resource(const char *name, const char *provider, const char *kind) { xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE); crm_xml_add(rsc, XML_ATTR_ID, name); crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, "ocf"); crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider); crm_xml_add(rsc, XML_ATTR_TYPE, kind); return rsc; } static void create_nvp(xmlNode *parent, const char *name, const char *value) { xmlNode *xml_nvp = create_xml_node(parent, XML_CIB_TAG_NVPAIR); crm_xml_set_id(xml_nvp, "%s-%s", ID(parent), name); crm_xml_add(xml_nvp, XML_NVPAIR_ATTR_NAME, name); crm_xml_add(xml_nvp, XML_NVPAIR_ATTR_VALUE, value); } static void create_op(xmlNode *parent, const char *prefix, const char *task, const char *interval) { xmlNode *xml_op = create_xml_node(parent, "op"); crm_xml_set_id(xml_op, "%s-%s-%s", prefix, task, interval); crm_xml_add(xml_op, XML_LRM_ATTR_INTERVAL, interval); crm_xml_add(xml_op, "name", task); } /*! * \internal * \brief Check whether cluster can manage resource inside container * * \param[in] data Container variant data * * \return TRUE if networking configuration is acceptable, FALSE otherwise * * \note The resource is manageable if an IP range or control port has been * specified. If a control port is used without an IP range, replicas per * host must be 1. */ static bool valid_network(container_variant_data_t *data) { if(data->ip_range_start) { return TRUE; } if(data->control_port) { if(data->replicas_per_host > 1) { pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix); data->replicas_per_host = 1; /* @TODO to be sure: clear_bit(rsc->flags, pe_rsc_unique); */ } return TRUE; } return FALSE; } static bool create_ip_resource( resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple, pe_working_set_t * data_set) { if(data->ip_range_start) { char *id = NULL; xmlNode *xml_ip = NULL; xmlNode *xml_obj = NULL; id = crm_strdup_printf("%s-ip-%s", data->prefix, tuple->ipaddr); crm_xml_sanitize_id(id); xml_ip = create_resource(id, "heartbeat", "IPaddr2"); free(id); xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS); crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset); create_nvp(xml_obj, "ip", tuple->ipaddr); if(data->host_network) { create_nvp(xml_obj, "nic", data->host_network); } if(data->host_netmask) { create_nvp(xml_obj, "cidr_netmask", data->host_netmask); } else { create_nvp(xml_obj, "cidr_netmask", "32"); } xml_obj = create_xml_node(xml_ip, "operations"); create_op(xml_obj, ID(xml_ip), "monitor", "60s"); // TODO: Other ops? Timeouts and intervals from underlying resource? if (common_unpack(xml_ip, &tuple->ip, parent, data_set) == false) { return FALSE; } parent->children = g_list_append(parent->children, tuple->ip); } return TRUE; } static bool create_docker_resource( resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple, pe_working_set_t * data_set) { int offset = 0, max = 4096; char *buffer = calloc(1, max+1); int doffset = 0, dmax = 1024; char *dbuffer = calloc(1, dmax+1); char *id = NULL; xmlNode *xml_docker = NULL; xmlNode *xml_obj = NULL; id = crm_strdup_printf("%s-docker-%d", data->prefix, tuple->offset); crm_xml_sanitize_id(id); xml_docker = create_resource(id, "heartbeat", "docker"); free(id); xml_obj = create_xml_node(xml_docker, XML_TAG_ATTR_SETS); crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset); create_nvp(xml_obj, "image", data->image); create_nvp(xml_obj, "allow_pull", "true"); create_nvp(xml_obj, "force_kill", "false"); create_nvp(xml_obj, "reuse", "false"); offset += snprintf(buffer+offset, max-offset, " --restart=no"); /* Set a container hostname only if we have an IP to map it to. * The user can set -h or --uts=host themselves if they want a nicer * name for logs, but this makes applications happy who need their * hostname to match the IP they bind to. */ if (data->ip_range_start != NULL) { offset += snprintf(buffer+offset, max-offset, " -h %s-%d", data->prefix, tuple->offset); } if(data->docker_network) { // offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s", tuple->ipaddr); offset += snprintf(buffer+offset, max-offset, " --net=%s", data->docker_network); } if(data->control_port) { offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port); } else { offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT); } for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) { container_mount_t *mount = pIter->data; if(mount->flags) { char *source = crm_strdup_printf( "%s/%s-%d", mount->source, data->prefix, tuple->offset); if(doffset > 0) { doffset += snprintf(dbuffer+doffset, dmax-doffset, ","); } doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source); offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target); free(source); } else { offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target); } if(mount->options) { offset += snprintf(buffer+offset, max-offset, ":%s", mount->options); } } for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) { container_port_t *port = pIter->data; if(tuple->ipaddr) { offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s", tuple->ipaddr, port->source, port->target); } else { offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target); } } if(data->docker_run_options) { offset += snprintf(buffer+offset, max-offset, " %s", data->docker_run_options); } if(data->docker_host_options) { offset += snprintf(buffer+offset, max-offset, " %s", data->docker_host_options); } create_nvp(xml_obj, "run_opts", buffer); free(buffer); create_nvp(xml_obj, "mount_points", dbuffer); free(dbuffer); if(tuple->child) { if(data->docker_run_command) { create_nvp(xml_obj, "run_cmd", data->docker_run_command); } else { create_nvp(xml_obj, "run_cmd", SBIN_DIR"/pacemaker_remoted"); } /* TODO: Allow users to specify their own? * * We just want to know if the container is alive, we'll * monitor the child independently */ create_nvp(xml_obj, "monitor_cmd", "/bin/true"); /* } else if(child && data->untrusted) { * Support this use-case? * * The ability to have resources started/stopped by us, but * unable to set attributes, etc. * * Arguably better to control API access this with ACLs like * "normal" remote nodes * * create_nvp(xml_obj, "run_cmd", "/usr/libexec/pacemaker/lrmd"); * create_nvp(xml_obj, "monitor_cmd", "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke"); */ } else { if(data->docker_run_command) { create_nvp(xml_obj, "run_cmd", data->docker_run_command); } /* TODO: Allow users to specify their own? * * We don't know what's in the container, so we just want * to know if it is alive */ create_nvp(xml_obj, "monitor_cmd", "/bin/true"); } xml_obj = create_xml_node(xml_docker, "operations"); create_op(xml_obj, ID(xml_docker), "monitor", "60s"); // TODO: Other ops? Timeouts and intervals from underlying resource? if (common_unpack(xml_docker, &tuple->docker, parent, data_set) == FALSE) { return FALSE; } parent->children = g_list_append(parent->children, tuple->docker); return TRUE; } /*! * \brief Ban a node from a resource's (and its children's) allowed nodes list * * \param[in,out] rsc Resource to modify * \param[in] uname Name of node to ban */ static void disallow_node(resource_t *rsc, const char *uname) { gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname); if (match) { ((pe_node_t *) match)->weight = -INFINITY; + ((pe_node_t *) match)->rsc_discover_mode = discover_never; } if (rsc->children) { GListPtr child; for (child = rsc->children; child != NULL; child = child->next) { disallow_node((resource_t *) (child->data), uname); } } } static bool create_remote_resource( resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple, pe_working_set_t * data_set) { if (tuple->child && valid_network(data)) { GHashTableIter gIter; GListPtr rsc_iter = NULL; node_t *node = NULL; xmlNode *xml_obj = NULL; xmlNode *xml_remote = NULL; char *id = crm_strdup_printf("%s-%d", data->prefix, tuple->offset); const char *uname = NULL; if (remote_id_conflict(id, data_set)) { free(id); // The biggest hammer we have id = crm_strdup_printf("pcmk-internal-%s-remote-%d", tuple->child->id, tuple->offset); CRM_ASSERT(remote_id_conflict(id, data_set) == FALSE); } xml_remote = create_resource(id, "pacemaker", "remote"); /* Abandon our created ID, and pull the copy from the XML, because we * need something that will get freed during data set cleanup to use as * the node ID and uname. */ free(id); id = NULL; uname = ID(xml_remote); xml_obj = create_xml_node(xml_remote, "operations"); create_op(xml_obj, uname, "monitor", "60s"); xml_obj = create_xml_node(xml_remote, XML_TAG_ATTR_SETS); crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, tuple->offset); if(tuple->ipaddr) { create_nvp(xml_obj, "addr", tuple->ipaddr); } else { // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside create_nvp(xml_obj, "addr", "#uname"); } if(data->control_port) { create_nvp(xml_obj, "port", data->control_port); } else { char *port_s = crm_itoa(DEFAULT_REMOTE_PORT); create_nvp(xml_obj, "port", port_s); free(port_s); } xml_obj = create_xml_node(xml_remote, XML_TAG_META_SETS); crm_xml_set_id(xml_obj, "%s-meta-%d", data->prefix, tuple->offset); create_nvp(xml_obj, XML_OP_ATTR_ALLOW_MIGRATE, "false"); /* This sets tuple->docker as tuple->remote's container, which is * similar to what happens with guest nodes. This is how the PE knows * that the bundle node is fenced by recovering docker, and that * remote should be ordered relative to docker. */ create_nvp(xml_obj, XML_RSC_ATTR_CONTAINER, tuple->docker->id); /* Ensure a node has been created for the guest (it may have already * been, if it has a permanent node attribute), and ensure its weight is * -INFINITY so no other resources can run on it. */ node = pe_find_node(data_set->nodes, uname); if (node == NULL) { node = pe_create_node(uname, uname, "remote", "-INFINITY", data_set); } else { node->weight = -INFINITY; } + node->rsc_discover_mode = discover_never; /* unpack_remote_nodes() ensures that each remote node and guest node * has a pe_node_t entry. Ideally, it would do the same for bundle nodes. * Unfortunately, a bundle has to be mostly unpacked before it's obvious * what nodes will be needed, so we do it just above. * * Worse, that means that the node may have been utilized while * unpacking other resources, without our weight correction. The most * likely place for this to happen is when common_unpack() calls * resource_location() to set a default score in symmetric clusters. * This adds a node *copy* to each resource's allowed nodes, and these * copies will have the wrong weight. * * As a hacky workaround, fix those copies here. * * @TODO Possible alternative: ensure bundles are unpacked before other * resources, so the weight is correct before any copies are made. */ for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) { disallow_node((resource_t *) (rsc_iter->data), uname); } tuple->node = node_copy(node); tuple->node->weight = 500; + tuple->node->rsc_discover_mode = discover_exclusive; if (common_unpack(xml_remote, &tuple->remote, parent, data_set) == FALSE) { return FALSE; } g_hash_table_iter_init(&gIter, tuple->remote->allowed_nodes); while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) { if(is_remote_node(node)) { /* Remote resources can only run on 'normal' cluster node */ node->weight = -INFINITY; } } tuple->node->details->remote_rsc = tuple->remote; /* #kind is irrelevant to bundles since it is only used in location * constraint rules, and those don't matter for resources inside * bundles. But just for clarity, a bundle is closer to "container" * (guest node) than the "remote" set by pe_create_node(). */ g_hash_table_insert(tuple->node->details->attrs, strdup("#kind"), strdup("container")); /* One effect of this is that setup_container() will add * tuple->remote to tuple->docker's fillers, which will make * rsc_contains_remote_node() true for tuple->docker. * * tuple->child does NOT get added to tuple->docker's fillers. * The only noticeable effect if it did would be for its fail count to * be taken into account when checking tuple->docker's migration * threshold. */ parent->children = g_list_append(parent->children, tuple->remote); } return TRUE; } static bool create_container( resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple, pe_working_set_t * data_set) { if(create_docker_resource(parent, data, tuple, data_set) == FALSE) { return TRUE; } if(create_ip_resource(parent, data, tuple, data_set) == FALSE) { return TRUE; } if(create_remote_resource(parent, data, tuple, data_set) == FALSE) { return TRUE; } if(tuple->child && tuple->ipaddr) { add_hash_param(tuple->child->meta, "external-ip", tuple->ipaddr); } if(tuple->remote) { /* * Allow the remote connection resource to be allocated to a * different node than the one on which the docker container * is active. * * Makes it possible to have remote nodes, running docker * containers with pacemaker_remoted inside in order to start * services inside those containers. */ set_bit(tuple->remote->flags, pe_rsc_allow_remote_remotes); } return FALSE; } static void mount_free(container_mount_t *mount) { free(mount->source); free(mount->target); free(mount->options); free(mount); } static void port_free(container_port_t *port) { free(port->source); free(port->target); free(port); } gboolean container_unpack(resource_t * rsc, pe_working_set_t * data_set) { const char *value = NULL; xmlNode *xml_obj = NULL; xmlNode *xml_resource = NULL; container_variant_data_t *container_data = NULL; CRM_ASSERT(rsc != NULL); pe_rsc_trace(rsc, "Processing resource %s...", rsc->id); container_data = calloc(1, sizeof(container_variant_data_t)); rsc->variant_opaque = container_data; container_data->prefix = strdup(rsc->id); xml_obj = first_named_child(rsc->xml, "docker"); if(xml_obj == NULL) { return FALSE; } value = crm_element_value(xml_obj, "masters"); container_data->masters = crm_parse_int(value, "0"); if (container_data->masters < 0) { pe_err("'masters' for %s must be nonnegative integer, using 0", rsc->id); container_data->masters = 0; } value = crm_element_value(xml_obj, "replicas"); if ((value == NULL) && (container_data->masters > 0)) { container_data->replicas = container_data->masters; } else { container_data->replicas = crm_parse_int(value, "1"); } if (container_data->replicas < 1) { pe_err("'replicas' for %s must be positive integer, using 1", rsc->id); container_data->replicas = 1; } /* * Communication between containers on the same host via the * floating IPs only works if docker is started with: * --userland-proxy=false --ip-masq=false */ value = crm_element_value(xml_obj, "replicas-per-host"); container_data->replicas_per_host = crm_parse_int(value, "1"); if (container_data->replicas_per_host < 1) { pe_err("'replicas-per-host' for %s must be positive integer, using 1", rsc->id); container_data->replicas_per_host = 1; } if (container_data->replicas_per_host == 1) { clear_bit(rsc->flags, pe_rsc_unique); } container_data->docker_run_command = crm_element_value_copy(xml_obj, "run-command"); container_data->docker_run_options = crm_element_value_copy(xml_obj, "options"); container_data->image = crm_element_value_copy(xml_obj, "image"); container_data->docker_network = crm_element_value_copy(xml_obj, "network"); xml_obj = first_named_child(rsc->xml, "network"); if(xml_obj) { container_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start"); container_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask"); container_data->host_network = crm_element_value_copy(xml_obj, "host-interface"); container_data->control_port = crm_element_value_copy(xml_obj, "control-port"); for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL; xml_child = __xml_next_element(xml_child)) { container_port_t *port = calloc(1, sizeof(container_port_t)); port->source = crm_element_value_copy(xml_child, "port"); if(port->source == NULL) { port->source = crm_element_value_copy(xml_child, "range"); } else { port->target = crm_element_value_copy(xml_child, "internal-port"); } if(port->source != NULL && strlen(port->source) > 0) { if(port->target == NULL) { port->target = strdup(port->source); } container_data->ports = g_list_append(container_data->ports, port); } else { pe_err("Invalid port directive %s", ID(xml_child)); port_free(port); } } } xml_obj = first_named_child(rsc->xml, "storage"); for (xmlNode *xml_child = __xml_first_child_element(xml_obj); xml_child != NULL; xml_child = __xml_next_element(xml_child)) { container_mount_t *mount = calloc(1, sizeof(container_mount_t)); mount->source = crm_element_value_copy(xml_child, "source-dir"); if(mount->source == NULL) { mount->source = crm_element_value_copy(xml_child, "source-dir-root"); mount->flags = 1; } mount->target = crm_element_value_copy(xml_child, "target-dir"); mount->options = crm_element_value_copy(xml_child, "options"); if(mount->source && mount->target) { container_data->mounts = g_list_append(container_data->mounts, mount); } else { pe_err("Invalid mount directive %s", ID(xml_child)); mount_free(mount); } } xml_obj = first_named_child(rsc->xml, "primitive"); if (xml_obj && valid_network(container_data)) { char *value = NULL; xmlNode *xml_set = NULL; if(container_data->masters > 0) { xml_resource = create_xml_node(NULL, XML_CIB_TAG_MASTER); } else { xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION); } crm_xml_set_id(xml_resource, "%s-%s", container_data->prefix, xml_resource->name); xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS); crm_xml_set_id(xml_set, "%s-%s-meta", container_data->prefix, xml_resource->name); create_nvp(xml_set, XML_RSC_ATTR_ORDERED, "true"); value = crm_itoa(container_data->replicas); create_nvp(xml_set, XML_RSC_ATTR_INCARNATION_MAX, value); free(value); value = crm_itoa(container_data->replicas_per_host); create_nvp(xml_set, XML_RSC_ATTR_INCARNATION_NODEMAX, value); free(value); if(container_data->replicas_per_host > 1) { create_nvp(xml_set, XML_RSC_ATTR_UNIQUE, "true"); } else { create_nvp(xml_set, XML_RSC_ATTR_UNIQUE, "false"); } if(container_data->masters) { value = crm_itoa(container_data->masters); create_nvp(xml_set, XML_RSC_ATTR_MASTER_MAX, value); free(value); } //crm_xml_add(xml_obj, XML_ATTR_ID, container_data->prefix); add_node_copy(xml_resource, xml_obj); } else if(xml_obj) { pe_err("Cannot control %s inside %s without either ip-range-start or control-port", rsc->id, ID(xml_obj)); return FALSE; } if(xml_resource) { int lpc = 0; GListPtr childIter = NULL; resource_t *new_rsc = NULL; container_mount_t *mount = NULL; container_port_t *port = NULL; int offset = 0, max = 1024; char *buffer = NULL; if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) { pe_err("Failed unpacking resource %s", ID(rsc->xml)); if (new_rsc != NULL && new_rsc->fns != NULL) { new_rsc->fns->free(new_rsc); } return FALSE; } container_data->child = new_rsc; mount = calloc(1, sizeof(container_mount_t)); mount->source = strdup(DEFAULT_REMOTE_KEY_LOCATION); mount->target = strdup(DEFAULT_REMOTE_KEY_LOCATION); mount->options = NULL; mount->flags = 0; container_data->mounts = g_list_append(container_data->mounts, mount); mount = calloc(1, sizeof(container_mount_t)); mount->source = strdup(CRM_LOG_DIR "/bundles"); mount->target = strdup("/var/log"); mount->options = NULL; mount->flags = 1; container_data->mounts = g_list_append(container_data->mounts, mount); port = calloc(1, sizeof(container_port_t)); if(container_data->control_port) { port->source = strdup(container_data->control_port); } else { port->source = crm_itoa(DEFAULT_REMOTE_PORT); } port->target = strdup(port->source); container_data->ports = g_list_append(container_data->ports, port); buffer = calloc(1, max+1); for(childIter = container_data->child->children; childIter != NULL; childIter = childIter->next) { container_grouping_t *tuple = calloc(1, sizeof(container_grouping_t)); tuple->child = childIter->data; tuple->offset = lpc++; // Ensure the child's notify gets set based on the underlying primitive's value if(is_set(tuple->child->flags, pe_rsc_notify)) { set_bit(container_data->child->flags, pe_rsc_notify); } offset += allocate_ip(container_data, tuple, buffer+offset, max-offset); container_data->tuples = g_list_append(container_data->tuples, tuple); container_data->attribute_target = g_hash_table_lookup(tuple->child->meta, XML_RSC_ATTR_TARGET); } container_data->docker_host_options = buffer; if(container_data->attribute_target) { g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET), strdup(container_data->attribute_target)); g_hash_table_replace(container_data->child->meta, strdup(XML_RSC_ATTR_TARGET), strdup(container_data->attribute_target)); } } else { // Just a naked container, no pacemaker-remote int offset = 0, max = 1024; char *buffer = calloc(1, max+1); for(int lpc = 0; lpc < container_data->replicas; lpc++) { container_grouping_t *tuple = calloc(1, sizeof(container_grouping_t)); tuple->offset = lpc; offset += allocate_ip(container_data, tuple, buffer+offset, max-offset); container_data->tuples = g_list_append(container_data->tuples, tuple); } container_data->docker_host_options = buffer; } for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { container_grouping_t *tuple = (container_grouping_t *)gIter->data; // TODO: Remove from list if create_container() returns TRUE create_container(rsc, container_data, tuple, data_set); } if(container_data->child) { rsc->children = g_list_append(rsc->children, container_data->child); } return TRUE; } static int tuple_rsc_active(resource_t *rsc, gboolean all) { if (rsc) { gboolean child_active = rsc->fns->active(rsc, all); if (child_active && !all) { return TRUE; } else if (!child_active && all) { return FALSE; } } return -1; } gboolean container_active(resource_t * rsc, gboolean all) { container_variant_data_t *container_data = NULL; GListPtr iter = NULL; get_container_variant_data(container_data, rsc); for (iter = container_data->tuples; iter != NULL; iter = iter->next) { container_grouping_t *tuple = (container_grouping_t *)(iter->data); int rsc_active; rsc_active = tuple_rsc_active(tuple->ip, all); if (rsc_active >= 0) { return (gboolean) rsc_active; } rsc_active = tuple_rsc_active(tuple->child, all); if (rsc_active >= 0) { return (gboolean) rsc_active; } rsc_active = tuple_rsc_active(tuple->docker, all); if (rsc_active >= 0) { return (gboolean) rsc_active; } rsc_active = tuple_rsc_active(tuple->remote, all); if (rsc_active >= 0) { return (gboolean) rsc_active; } } /* If "all" is TRUE, we've already checked that no resources were inactive, * so return TRUE; if "all" is FALSE, we didn't find any active resources, * so return FALSE. */ return all; } resource_t * find_container_child(const char *stem, resource_t * rsc, node_t *node) { container_variant_data_t *container_data = NULL; resource_t *parent = uber_parent(rsc); CRM_ASSERT(parent->parent); parent = parent->parent; get_container_variant_data(container_data, parent); if (is_not_set(rsc->flags, pe_rsc_unique)) { for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { container_grouping_t *tuple = (container_grouping_t *)gIter->data; CRM_ASSERT(tuple); if(tuple->node->details == node->details) { rsc = tuple->child; break; } } } if (rsc && safe_str_neq(stem, rsc->id)) { free(rsc->clone_name); rsc->clone_name = strdup(stem); } return rsc; } static void print_rsc_in_list(resource_t *rsc, const char *pre_text, long options, void *print_data) { if (rsc != NULL) { if (options & pe_print_html) { status_print("
  • "); } rsc->fns->print(rsc, pre_text, options, print_data); if (options & pe_print_html) { status_print("
    • \n"); } } } static void container_print_xml(resource_t * rsc, const char *pre_text, long options, void *print_data) { container_variant_data_t *container_data = NULL; char *child_text = NULL; CRM_CHECK(rsc != NULL, return); if (pre_text == NULL) { pre_text = ""; } child_text = crm_concat(pre_text, " ", ' '); get_container_variant_data(container_data, rsc); status_print("%sid); status_print("type=\"docker\" "); status_print("image=\"%s\" ", container_data->image); status_print("unique=\"%s\" ", is_set(rsc->flags, pe_rsc_unique)? "true" : "false"); status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false"); status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false"); status_print(">\n"); for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { container_grouping_t *tuple = (container_grouping_t *)gIter->data; CRM_ASSERT(tuple); status_print("%s \n", pre_text, tuple->offset); print_rsc_in_list(tuple->ip, child_text, options, print_data); print_rsc_in_list(tuple->child, child_text, options, print_data); print_rsc_in_list(tuple->docker, child_text, options, print_data); print_rsc_in_list(tuple->remote, child_text, options, print_data); status_print("%s \n", pre_text); } status_print("%s\n", pre_text); free(child_text); } static void tuple_print(container_grouping_t * tuple, const char *pre_text, long options, void *print_data) { node_t *node = NULL; resource_t *rsc = tuple->child; int offset = 0; char buffer[LINE_MAX]; if(rsc == NULL) { rsc = tuple->docker; } if(tuple->remote) { offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_printable_id(tuple->remote)); } else { offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_printable_id(tuple->docker)); } if(tuple->ipaddr) { offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)", tuple->ipaddr); } if(tuple->docker && tuple->docker->running_on != NULL) { node = tuple->docker->running_on->data; } else if (tuple->docker == NULL && rsc->running_on != NULL) { node = rsc->running_on->data; } common_print(rsc, pre_text, buffer, node, options, print_data); } void container_print(resource_t * rsc, const char *pre_text, long options, void *print_data) { container_variant_data_t *container_data = NULL; char *child_text = NULL; CRM_CHECK(rsc != NULL, return); if (options & pe_print_xml) { container_print_xml(rsc, pre_text, options, print_data); return; } get_container_variant_data(container_data, rsc); if (pre_text == NULL) { pre_text = " "; } status_print("%sDocker container%s: %s [%s]%s%s\n", pre_text, container_data->replicas>1?" set":"", rsc->id, container_data->image, is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "", is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)"); if (options & pe_print_html) { status_print("
    \n
      \n"); } for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { container_grouping_t *tuple = (container_grouping_t *)gIter->data; CRM_ASSERT(tuple); if (options & pe_print_html) { status_print("
    • "); } if(is_set(options, pe_print_clone_details)) { child_text = crm_strdup_printf(" %s", pre_text); if(g_list_length(container_data->tuples) > 1) { status_print(" %sReplica[%d]\n", pre_text, tuple->offset); } if (options & pe_print_html) { status_print("
      \n
        \n"); } print_rsc_in_list(tuple->ip, child_text, options, print_data); print_rsc_in_list(tuple->docker, child_text, options, print_data); print_rsc_in_list(tuple->remote, child_text, options, print_data); print_rsc_in_list(tuple->child, child_text, options, print_data); if (options & pe_print_html) { status_print("
      \n"); } } else { child_text = crm_strdup_printf("%s ", pre_text); tuple_print(tuple, child_text, options, print_data); } free(child_text); if (options & pe_print_html) { status_print("
      • \n"); } } if (options & pe_print_html) { status_print("
    \n"); } } void tuple_free(container_grouping_t *tuple) { if(tuple == NULL) { return; } if(tuple->node) { free(tuple->node); tuple->node = NULL; } if(tuple->ip) { free_xml(tuple->ip->xml); tuple->ip->xml = NULL; tuple->ip->fns->free(tuple->ip); tuple->ip->xml = NULL; free_xml(tuple->ip->xml); tuple->ip = NULL; } if(tuple->docker) { free_xml(tuple->docker->xml); tuple->docker->xml = NULL; tuple->docker->fns->free(tuple->docker); tuple->docker = NULL; } if(tuple->remote) { free_xml(tuple->remote->xml); tuple->remote->xml = NULL; tuple->remote->fns->free(tuple->remote); tuple->remote = NULL; } free(tuple->ipaddr); free(tuple); } void container_free(resource_t * rsc) { container_variant_data_t *container_data = NULL; CRM_CHECK(rsc != NULL, return); get_container_variant_data(container_data, rsc); pe_rsc_trace(rsc, "Freeing %s", rsc->id); free(container_data->prefix); free(container_data->image); free(container_data->control_port); free(container_data->host_network); free(container_data->host_netmask); free(container_data->ip_range_start); free(container_data->docker_network); free(container_data->docker_run_options); free(container_data->docker_run_command); free(container_data->docker_host_options); g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free); g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free); g_list_free_full(container_data->ports, (GDestroyNotify)port_free); g_list_free(rsc->children); if(container_data->child) { free_xml(container_data->child->xml); container_data->child->xml = NULL; container_data->child->fns->free(container_data->child); } common_free(rsc); } enum rsc_role_e container_resource_state(const resource_t * rsc, gboolean current) { enum rsc_role_e container_role = RSC_ROLE_UNKNOWN; return container_role; } diff --git a/pengine/allocate.c b/pengine/allocate.c index d610035b07..401c18ec63 100644 --- a/pengine/allocate.c +++ b/pengine/allocate.c @@ -1,2498 +1,2497 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include CRM_TRACE_INIT_DATA(pe_allocate); void set_alloc_actions(pe_working_set_t * data_set); void migrate_reload_madness(pe_working_set_t * data_set); extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set); extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set); static void apply_remote_node_ordering(pe_working_set_t *data_set); static enum remote_connection_state get_remote_node_state(pe_node_t *node); enum remote_connection_state { remote_state_unknown = 0, remote_state_alive = 1, remote_state_resting = 2, remote_state_failed = 3, remote_state_stopped = 4 }; resource_alloc_functions_t resource_class_alloc_functions[] = { { native_merge_weights, native_color, native_create_actions, native_create_probe, native_internal_constraints, native_rsc_colocation_lh, native_rsc_colocation_rh, native_rsc_location, native_action_flags, native_update_actions, native_expand, native_append_meta, }, { group_merge_weights, group_color, group_create_actions, native_create_probe, group_internal_constraints, group_rsc_colocation_lh, group_rsc_colocation_rh, group_rsc_location, group_action_flags, group_update_actions, group_expand, group_append_meta, }, { clone_merge_weights, clone_color, clone_create_actions, clone_create_probe, clone_internal_constraints, clone_rsc_colocation_lh, clone_rsc_colocation_rh, clone_rsc_location, clone_action_flags, container_update_actions, clone_expand, clone_append_meta, }, { master_merge_weights, master_color, master_create_actions, clone_create_probe, master_internal_constraints, clone_rsc_colocation_lh, master_rsc_colocation_rh, clone_rsc_location, clone_action_flags, container_update_actions, clone_expand, master_append_meta, }, { container_merge_weights, container_color, container_create_actions, container_create_probe, container_internal_constraints, container_rsc_colocation_lh, container_rsc_colocation_rh, container_rsc_location, container_action_flags, container_update_actions, container_expand, container_append_meta, } }; gboolean update_action_flags(action_t * action, enum pe_action_flags flags, const char *source, int line) { static unsigned long calls = 0; gboolean changed = FALSE; gboolean clear = is_set(flags, pe_action_clear); enum pe_action_flags last = action->flags; if (clear) { action->flags = crm_clear_bit(source, line, action->uuid, action->flags, flags); } else { action->flags = crm_set_bit(source, line, action->uuid, action->flags, flags); } if (last != action->flags) { calls++; changed = TRUE; /* Useful for tracking down _who_ changed a specific flag */ /* CRM_ASSERT(calls != 534); */ clear_bit(flags, pe_action_clear); crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)", action->uuid, action->node ? action->node->details->uname : "[none]", clear ? "un-" : "", flags, last, action->flags, calls, source); } return changed; } static gboolean check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry, gboolean active_here, pe_working_set_t * data_set) { int attr_lpc = 0; gboolean force_restart = FALSE; gboolean delete_resource = FALSE; gboolean changed = FALSE; const char *value = NULL; const char *old_value = NULL; const char *attr_list[] = { XML_ATTR_TYPE, XML_AGENT_ATTR_CLASS, XML_AGENT_ATTR_PROVIDER }; for (; attr_lpc < DIMOF(attr_list); attr_lpc++) { value = crm_element_value(rsc->xml, attr_list[attr_lpc]); old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]); if (value == old_value /* ie. NULL */ || crm_str_eq(value, old_value, TRUE)) { continue; } changed = TRUE; trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set); if (active_here) { force_restart = TRUE; crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s", rsc->id, node->details->uname, attr_list[attr_lpc], crm_str(old_value), crm_str(value)); } } if (force_restart) { /* make sure the restart happens */ stop_action(rsc, node, FALSE); set_bit(rsc->flags, pe_rsc_start_pending); delete_resource = TRUE; } else if (changed) { delete_resource = TRUE; } return delete_resource; } static void CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node, const char *reason, pe_working_set_t * data_set) { int interval = 0; action_t *cancel = NULL; char *key = NULL; const char *task = NULL; const char *call_id = NULL; const char *interval_s = NULL; CRM_CHECK(xml_op != NULL, return); CRM_CHECK(active_node != NULL, return); task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID); interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); interval = crm_parse_int(interval_s, "0"); /* we need to reconstruct the key because of the way we used to construct resource IDs */ key = generate_op_key(rsc->id, task, interval); crm_info("Action %s on %s will be stopped: %s", key, active_node->details->uname, reason ? reason : "unknown"); /* TODO: This looks highly dangerous if we ever try to schedule 'key' too */ cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set); free(cancel->task); free(cancel->cancel_task); cancel->task = strdup(RSC_CANCEL); cancel->cancel_task = strdup(task); add_hash_param(cancel->meta, XML_LRM_ATTR_TASK, task); add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id); add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s); custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set); free(key); key = NULL; } static gboolean check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op, pe_working_set_t * data_set) { char *key = NULL; int interval = 0; const char *interval_s = NULL; const op_digest_cache_t *digest_data = NULL; gboolean did_change = FALSE; const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); const char *op_version; const char *digest_secure = NULL; CRM_CHECK(active_node != NULL, return FALSE); if (safe_str_eq(task, RSC_STOP)) { return FALSE; } interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); interval = crm_parse_int(interval_s, "0"); if (interval > 0) { xmlNode *op_match = NULL; /* we need to reconstruct the key because of the way we used to construct resource IDs */ key = generate_op_key(rsc->id, task, interval); pe_rsc_trace(rsc, "Checking parameters for %s", key); op_match = find_rsc_op_entry(rsc, key); if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) { CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set); free(key); return TRUE; } else if (op_match == NULL) { pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname); free(key); return TRUE; } free(key); key = NULL; } crm_trace("Testing %s_%s_%d on %s", rsc->id, task, interval, active_node->details->uname); if (interval == 0 && safe_str_eq(task, RSC_STATUS)) { /* Reload based on the start action not a probe */ task = RSC_START; } else if (interval == 0 && safe_str_eq(task, RSC_MIGRATED)) { /* Reload based on the start action not a migrate */ task = RSC_START; } else if (interval == 0 && safe_str_eq(task, RSC_PROMOTE)) { /* Reload based on the start action not a promote */ task = RSC_START; } op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION); digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set); if(is_set(data_set->flags, pe_flag_sanitized)) { digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST); } if(digest_data->rc != RSC_DIGEST_MATCH && digest_secure && digest_data->digest_secure_calc && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) { fprintf(stdout, "Only 'private' parameters to %s_%s_%d on %s changed: %s\n", rsc->id, task, interval, active_node->details->uname, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); } else if (digest_data->rc == RSC_DIGEST_RESTART) { /* Changes that force a restart */ const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST); did_change = TRUE; key = generate_op_key(rsc->id, task, interval); crm_log_xml_info(digest_data->params_restart, "params:restart"); pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s", key, active_node->details->uname, crm_str(digest_restart), digest_data->digest_restart_calc, op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set); trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set); } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) { /* Changes that can potentially be handled by a reload */ const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST); const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); did_change = TRUE; trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set); crm_log_xml_info(digest_data->params_all, "params:reload"); key = generate_op_key(rsc->id, task, interval); pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s", key, active_node->details->uname, crm_str(digest_all), digest_data->digest_all_calc, op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); if (interval > 0) { action_t *op = NULL; #if 0 /* Always reload/restart the entire resource */ ReloadRsc(rsc, active_node, data_set); #else /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */ op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set); set_bit(op->flags, pe_action_reschedule); #endif } else if (digest_restart && rsc->isolation_wrapper == NULL && (uber_parent(rsc))->isolation_wrapper == NULL) { pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id); /* Reload this resource */ ReloadRsc(rsc, active_node, data_set); free(key); } else { pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id); /* Re-send the start/demote/promote op * Recurring ops will be detected independently */ custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set); } } return did_change; } static void check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set) { GListPtr gIter = NULL; int offset = -1; int interval = 0; int stop_index = 0; int start_index = 0; const char *task = NULL; const char *interval_s = NULL; xmlNode *rsc_op = NULL; GListPtr op_list = NULL; GListPtr sorted_op_list = NULL; gboolean is_probe = FALSE; gboolean did_change = FALSE; CRM_CHECK(node != NULL, return); if (is_set(rsc->flags, pe_rsc_orphan)) { resource_t *parent = uber_parent(rsc); if(parent == NULL || pe_rsc_is_clone(parent) == FALSE || is_set(parent->flags, pe_rsc_unique)) { pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id); DeleteRsc(rsc, node, FALSE, data_set); } else { pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id); } return; } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) { if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) { DeleteRsc(rsc, node, FALSE, data_set); } pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s", rsc->id, node->details->uname); return; } pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname); if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) { DeleteRsc(rsc, node, FALSE, data_set); } for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { op_list = g_list_prepend(op_list, rsc_op); } } sorted_op_list = g_list_sort(op_list, sort_op_by_callid); calculate_active_ops(sorted_op_list, &start_index, &stop_index); for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) { xmlNode *rsc_op = (xmlNode *) gIter->data; offset++; if (start_index < stop_index) { /* stopped */ continue; } else if (offset < start_index) { /* action occurred prior to a start */ continue; } is_probe = FALSE; did_change = FALSE; task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK); interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL); interval = crm_parse_int(interval_s, "0"); if (interval == 0 && safe_str_eq(task, RSC_STATUS)) { is_probe = TRUE; } if (interval > 0 && (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) { CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set); } else if (is_probe || safe_str_eq(task, RSC_START) || safe_str_eq(task, RSC_PROMOTE) || interval > 0 || safe_str_eq(task, RSC_MIGRATED)) { did_change = check_action_definition(rsc, node, rsc_op, data_set); } if (did_change && get_failcount(node, rsc, NULL, data_set)) { char *key = NULL; action_t *action_clear = NULL; key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0); action_clear = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set); set_bit(action_clear->flags, pe_action_runnable); crm_notice("Clearing failure of %s on %s " "because action definition changed " CRM_XS " %s", rsc->id, node->details->uname, action_clear->uuid); } } g_list_free(sorted_op_list); } static GListPtr find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones, gboolean partial, pe_working_set_t * data_set) { GListPtr gIter = NULL; gboolean match = FALSE; if (id == NULL) { return NULL; } else if (rsc == NULL && data_set) { for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; result = find_rsc_list(result, child, id, renamed_clones, partial, NULL); } return result; } else if (rsc == NULL) { return NULL; } if (partial) { if (strstr(rsc->id, id)) { match = TRUE; } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) { match = TRUE; } } else { if (strcmp(rsc->id, id) == 0) { match = TRUE; } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) { match = TRUE; } } if (match) { result = g_list_prepend(result, rsc); } if (rsc->children) { gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; result = find_rsc_list(result, child, id, renamed_clones, partial, NULL); } } return result; } static void check_actions(pe_working_set_t * data_set) { const char *id = NULL; node_t *node = NULL; xmlNode *lrm_rscs = NULL; xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input); xmlNode *node_state = NULL; for (node_state = __xml_first_child(status); node_state != NULL; node_state = __xml_next_element(node_state)) { if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) { id = crm_element_value(node_state, XML_ATTR_ID); lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE); node = pe_find_node_id(data_set->nodes, id); if (node == NULL) { continue; /* Still need to check actions for a maintenance node to cancel existing monitor operations */ } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) { crm_trace("Skipping param check for %s: can't run resources", node->details->uname); continue; } crm_trace("Processing node %s", node->details->uname); if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) { xmlNode *rsc_entry = NULL; for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL; rsc_entry = __xml_next_element(rsc_entry)) { if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { if (xml_has_children(rsc_entry)) { GListPtr gIter = NULL; GListPtr result = NULL; const char *rsc_id = ID(rsc_entry); CRM_CHECK(rsc_id != NULL, return); result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set); for (gIter = result; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; if (rsc->variant != pe_native) { continue; } check_actions_for(rsc_entry, rsc, node, data_set); } g_list_free(result); } } } } } } } static gboolean apply_placement_constraints(pe_working_set_t * data_set) { GListPtr gIter = NULL; crm_trace("Applying constraints..."); for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) { rsc_to_node_t *cons = (rsc_to_node_t *) gIter->data; cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons); } return TRUE; } static gboolean failcount_clear_action_exists(node_t * node, resource_t * rsc) { gboolean rc = FALSE; char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0); GListPtr list = find_actions_exact(rsc->actions, key, node); if (list) { rc = TRUE; } g_list_free(list); free(key); return rc; } /*! * \internal * \brief Force resource away if failures hit migration threshold * * \param[in,out] rsc Resource to check for failures * \param[in,out] node Node to check for failures * \param[in,out] data_set Cluster working set to update */ static void check_migration_threshold(resource_t *rsc, node_t *node, pe_working_set_t *data_set) { int fail_count, countdown; resource_t *failed; /* Migration threshold of 0 means never force away */ if (rsc->migration_threshold == 0) { return; } /* If there are no failures, there's no need to force away */ fail_count = get_failcount_all(node, rsc, NULL, data_set); if (fail_count <= 0) { return; } /* How many more times recovery will be tried on this node */ countdown = QB_MAX(rsc->migration_threshold - fail_count, 0); /* If failed resource has a parent, we'll force the parent away */ failed = rsc; if (is_not_set(rsc->flags, pe_rsc_unique)) { failed = uber_parent(rsc); } if (countdown == 0) { resource_location(failed, node, -INFINITY, "__fail_limit__", data_set); crm_warn("Forcing %s away from %s after %d failures (max=%d)", failed->id, node->details->uname, fail_count, rsc->migration_threshold); } else { crm_info("%s can fail %d more times on %s before being forced off", failed->id, countdown, node->details->uname); } } static void common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set) { if (rsc->children) { GListPtr gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; common_apply_stickiness(child_rsc, node, data_set); } return; } if (is_set(rsc->flags, pe_rsc_managed) && rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) { node_t *current = pe_find_node_id(rsc->running_on, node->details->id); node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id); if (current == NULL) { } else if (match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) { resource_t *sticky_rsc = rsc; resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set); pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location" " (node=%s, weight=%d)", sticky_rsc->id, node->details->uname, rsc->stickiness); } else { GHashTableIter iter; node_t *nIter = NULL; pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric" " and node %s is not explicitly allowed", rsc->id, node->details->uname); g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) { crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight); } } } /* Check the migration threshold only if a failcount clear action * has not already been placed for this resource on the node. * There is no sense in potentially forcing the resource from this * node if the failcount is being reset anyway. */ if (failcount_clear_action_exists(node, rsc) == FALSE) { check_migration_threshold(rsc, node, data_set); } } void complex_set_cmds(resource_t * rsc) { GListPtr gIter = rsc->children; rsc->cmds = &resource_class_alloc_functions[rsc->variant]; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; complex_set_cmds(child_rsc); } } void set_alloc_actions(pe_working_set_t * data_set) { GListPtr gIter = data_set->resources; for (; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; complex_set_cmds(rsc); } } static void calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data) { const char *key = (const char *)gKey; const char *value = (const char *)gValue; int *system_health = (int *)user_data; if (!gKey || !gValue || !user_data) { return; } /* Does it start with #health? */ if (0 == strncmp(key, "#health", 7)) { int score; /* Convert the value into an integer */ score = char2score(value); /* Add it to the running total */ *system_health = merge_weights(score, *system_health); } } static gboolean apply_system_health(pe_working_set_t * data_set) { GListPtr gIter = NULL; const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy"); int base_health = 0; if (health_strategy == NULL || safe_str_eq(health_strategy, "none")) { /* Prevent any accidental health -> score translation */ node_score_red = 0; node_score_yellow = 0; node_score_green = 0; return TRUE; } else if (safe_str_eq(health_strategy, "migrate-on-red")) { /* Resources on nodes which have health values of red are * weighted away from that node. */ node_score_red = -INFINITY; node_score_yellow = 0; node_score_green = 0; } else if (safe_str_eq(health_strategy, "only-green")) { /* Resources on nodes which have health values of red or yellow * are forced away from that node. */ node_score_red = -INFINITY; node_score_yellow = -INFINITY; node_score_green = 0; } else if (safe_str_eq(health_strategy, "progressive")) { /* Same as the above, but use the r/y/g scores provided by the user * Defaults are provided by the pe_prefs table * Also, custom health "base score" can be used */ base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0"); } else if (safe_str_eq(health_strategy, "custom")) { /* Requires the admin to configure the rsc_location constaints for * processing the stored health scores */ /* TODO: Check for the existence of appropriate node health constraints */ return TRUE; } else { crm_err("Unknown node health strategy: %s", health_strategy); return FALSE; } crm_info("Applying automated node health strategy: %s", health_strategy); for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { int system_health = base_health; node_t *node = (node_t *) gIter->data; /* Search through the node hash table for system health entries. */ g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health); crm_info(" Node %s has an combined system health of %d", node->details->uname, system_health); /* If the health is non-zero, then create a new rsc2node so that the * weight will be added later on. */ if (system_health != 0) { GListPtr gIter2 = data_set->resources; for (; gIter2 != NULL; gIter2 = gIter2->next) { resource_t *rsc = (resource_t *) gIter2->data; rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set); } } } return TRUE; } gboolean stage0(pe_working_set_t * data_set) { xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input); if (data_set->input == NULL) { return FALSE; } if (is_set(data_set->flags, pe_flag_have_status) == FALSE) { crm_trace("Calculating status"); cluster_status(data_set); } set_alloc_actions(data_set); apply_system_health(data_set); unpack_constraints(cib_constraints, data_set); return TRUE; } /* * Check nodes for resources started outside of the LRM */ gboolean probe_resources(pe_working_set_t * data_set) { action_t *probe_node_complete = NULL; for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; const char *probed = node_attribute_raw(node, CRM_OP_PROBED); if (is_container_remote_node(node)) { - /* TODO enable guest node probes once ordered probing is implemented */ - continue; + /* Guest node probes and their ordering requirements are now functional */ } else if (node->details->online == FALSE && node->details->remote_rsc) { // TODO figure out why this results in fence loop /* enum remote_connection_state state = get_remote_node_state(node); if(state == remote_state_failed) { pe_fence_node(data_set, node, "the connection is unrecoverable"); } */ continue; } else if(node->details->online == FALSE) { continue; } else if (node->details->unclean) { continue; } else if (node->details->rsc_discovery_enabled == FALSE) { /* resource discovery is disabled for this node */ continue; } if (probed != NULL && crm_is_true(probed) == FALSE) { action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname), CRM_OP_REPROBE, node, FALSE, TRUE, data_set); add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); continue; } for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) { resource_t *rsc = (resource_t *) gIter2->data; rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set); } } return TRUE; } static void rsc_discover_filter(resource_t *rsc, node_t *node) { GListPtr gIter = rsc->children; resource_t *top = uber_parent(rsc); node_t *match; if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) { return; } for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; rsc_discover_filter(child_rsc, node); } match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); if (match && match->rsc_discover_mode != discover_exclusive) { match->weight = -INFINITY; } } /* * Count how many valid nodes we have (so we know the maximum number of * colors we can resolve). * * Apply node constraints (ie. filter the "allowed_nodes" part of resources */ gboolean stage2(pe_working_set_t * data_set) { GListPtr gIter = NULL; crm_trace("Applying placement constraints"); gIter = data_set->nodes; for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; if (node == NULL) { /* error */ } else if (node->weight >= 0.0 /* global weight */ && node->details->online && node->details->type != node_ping) { data_set->max_valid_nodes++; } } apply_placement_constraints(data_set); gIter = data_set->nodes; for (; gIter != NULL; gIter = gIter->next) { GListPtr gIter2 = NULL; node_t *node = (node_t *) gIter->data; gIter2 = data_set->resources; for (; gIter2 != NULL; gIter2 = gIter2->next) { resource_t *rsc = (resource_t *) gIter2->data; common_apply_stickiness(rsc, node, data_set); rsc_discover_filter(rsc, node); } } return TRUE; } /* * Create internal resource constraints before allocation */ gboolean stage3(pe_working_set_t * data_set) { GListPtr gIter = data_set->resources; for (; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; rsc->cmds->internal_constraints(rsc, data_set); } return TRUE; } /* * Check for orphaned or redefined actions */ gboolean stage4(pe_working_set_t * data_set) { check_actions(data_set); return TRUE; } static gint sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data) { int rc = 0; int r1_weight = -INFINITY; int r2_weight = -INFINITY; const char *reason = "existence"; const GListPtr nodes = (GListPtr) data; resource_t *resource1 = (resource_t *) convert_const_pointer(a); resource_t *resource2 = (resource_t *) convert_const_pointer(b); node_t *r1_node = NULL; node_t *r2_node = NULL; GListPtr gIter = NULL; GHashTable *r1_nodes = NULL; GHashTable *r2_nodes = NULL; if (a == NULL && b == NULL) { goto done; } if (a == NULL) { return 1; } if (b == NULL) { return -1; } reason = "priority"; r1_weight = resource1->priority; r2_weight = resource2->priority; if (r1_weight > r2_weight) { rc = -1; goto done; } if (r1_weight < r2_weight) { rc = 1; goto done; } reason = "no node list"; if (nodes == NULL) { goto done; } r1_nodes = rsc_merge_weights(resource1, resource1->id, NULL, NULL, 1, pe_weights_forward | pe_weights_init); dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes); r2_nodes = rsc_merge_weights(resource2, resource2->id, NULL, NULL, 1, pe_weights_forward | pe_weights_init); dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes); /* Current location score */ reason = "current location"; r1_weight = -INFINITY; r2_weight = -INFINITY; if (resource1->running_on) { r1_node = g_list_nth_data(resource1->running_on, 0); r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id); if (r1_node != NULL) { r1_weight = r1_node->weight; } } if (resource2->running_on) { r2_node = g_list_nth_data(resource2->running_on, 0); r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id); if (r2_node != NULL) { r2_weight = r2_node->weight; } } if (r1_weight > r2_weight) { rc = -1; goto done; } if (r1_weight < r2_weight) { rc = 1; goto done; } reason = "score"; for (gIter = nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; r1_node = NULL; r2_node = NULL; r1_weight = -INFINITY; if (r1_nodes) { r1_node = g_hash_table_lookup(r1_nodes, node->details->id); } if (r1_node) { r1_weight = r1_node->weight; } r2_weight = -INFINITY; if (r2_nodes) { r2_node = g_hash_table_lookup(r2_nodes, node->details->id); } if (r2_node) { r2_weight = r2_node->weight; } if (r1_weight > r2_weight) { rc = -1; goto done; } if (r1_weight < r2_weight) { rc = 1; goto done; } } done: crm_trace("%s (%d) on %s %c %s (%d) on %s: %s", resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a", rc < 0 ? '>' : rc > 0 ? '<' : '=', resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason); if (r1_nodes) { g_hash_table_destroy(r1_nodes); } if (r2_nodes) { g_hash_table_destroy(r2_nodes); } return rc; } static void allocate_resources(pe_working_set_t * data_set) { GListPtr gIter = NULL; if (is_set(data_set->flags, pe_flag_have_remote_nodes)) { /* Force remote connection resources to be allocated first. This * also forces any colocation dependencies to be allocated as well */ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; if (rsc->is_remote_node == FALSE) { continue; } pe_rsc_trace(rsc, "Allocating: %s", rsc->id); /* For remote node connection resources, always prefer the partial * migration target during resource allocation, if the rsc is in the * middle of a migration. */ rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set); } } /* now do the rest of the resources */ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; if (rsc->is_remote_node == TRUE) { continue; } pe_rsc_trace(rsc, "Allocating: %s", rsc->id); rsc->cmds->allocate(rsc, NULL, data_set); } } static void cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set) { GListPtr gIter = NULL; if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) { return; } /* Don't recurse into ->children, those are just unallocated clone instances */ if(is_not_set(rsc->flags, pe_rsc_orphan)) { return; } for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; if (node->details->online && get_failcount(node, rsc, NULL, data_set)) { char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0); action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set); add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); pe_rsc_info(rsc, "Clearing failure of %s on %s because it is orphaned " CRM_XS " %s", rsc->id, node->details->uname, clear_op->uuid); custom_action_order(rsc, NULL, clear_op, rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_optional, data_set); } } } gboolean stage5(pe_working_set_t * data_set) { GListPtr gIter = NULL; if (safe_str_neq(data_set->placement_strategy, "default")) { GListPtr nodes = g_list_copy(data_set->nodes); nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL); data_set->resources = g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes); g_list_free(nodes); } gIter = data_set->nodes; for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Original", node); } crm_trace("Allocating services"); /* Take (next) highest resource, assign it and create its actions */ allocate_resources(data_set); gIter = data_set->nodes; for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node); } if (is_set(data_set->flags, pe_flag_startup_probes)) { crm_trace("Calculating needed probes"); /* This code probably needs optimization * ptest -x with 100 nodes, 100 clones and clone-max=100: With probes: ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints 36s ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph Without probes: ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph */ probe_resources(data_set); } crm_trace("Handle orphans"); for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; cleanup_orphans(rsc, data_set); } crm_trace("Creating actions"); for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; rsc->cmds->create_actions(rsc, data_set); } crm_trace("Creating done"); return TRUE; } static gboolean is_managed(const resource_t * rsc) { GListPtr gIter = rsc->children; if (is_set(rsc->flags, pe_rsc_managed)) { return TRUE; } for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; if (is_managed(child_rsc)) { return TRUE; } } return FALSE; } static gboolean any_managed_resources(pe_working_set_t * data_set) { GListPtr gIter = data_set->resources; for (; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; if (is_managed(rsc)) { return TRUE; } } return FALSE; } /*! * \internal * \brief Create pseudo-op for guest node fence, and order relative to it * * \param[in] node Guest node to fence * \param[in] done STONITH_DONE operation * \param[in] data_set Working set of CIB state */ static void fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set) { resource_t *container = node->details->remote_rsc->container; pe_action_t *stop = NULL; pe_action_t *stonith_op = NULL; /* The fence action is just a label; we don't do anything differently for * off vs. reboot. We specify it explicitly, rather than let it default to * cluster's default action, because we are not _initiating_ fencing -- we * are creating a pseudo-event to describe fencing that is already occurring * by other means (container recovery). */ const char *fence_action = "off"; /* Check whether guest's container resource is has any explicit stop or * start (the stop may be implied by fencing of the guest's host). */ if (container) { stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL); if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) { fence_action = "reboot"; } } /* Create a fence pseudo-event, so we have an event to order actions * against, and crmd can always detect it. */ stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set); update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); /* We want to imply stops/demotes after the guest is stopped, not wait until * it is restarted, so we always order pseudo-fencing after stop, not start * (even though start might be closer to what is done for a real reboot). */ if(stop && is_set(stop->flags, pe_action_pseudo)) { pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set); crm_info("Implying guest node %s is down (action %d) after %s fencing", node->details->uname, stonith_op->id, stop->node->details->uname); order_actions(parent_stonith_op, stonith_op, pe_order_runnable_left|pe_order_implies_then); } else if (stop) { order_actions(stop, stonith_op, pe_order_runnable_left|pe_order_implies_then); crm_info("Implying guest node %s is down (action %d) " "after container %s is stopped (action %d)", node->details->uname, stonith_op->id, container->id, stop->id); } else { crm_info("Implying guest node %s is down (action %d) ", node->details->uname, stonith_op->id); } /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */ /* Order/imply other actions relative to pseudo-fence as with real fence */ stonith_constraints(node, stonith_op, data_set); order_actions(stonith_op, done, pe_order_implies_then); } /* * Create dependencies for stonith and shutdown operations */ gboolean stage6(pe_working_set_t * data_set) { action_t *dc_down = NULL; action_t *dc_fence = NULL; action_t *stonith_op = NULL; action_t *last_stonith = NULL; gboolean integrity_lost = FALSE; action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set); action_t *done = get_pseudo_op(STONITH_DONE, data_set); gboolean need_stonith = TRUE; GListPtr gIter; GListPtr stonith_ops = NULL; /* Remote ordering constraints need to happen prior to calculate * fencing because it is one more place we will mark the node as * dirty. * * A nice side-effect of doing it first is that we can remove a * bunch of special logic from apply_*_ordering() because its * already part of pe_fence_node() */ crm_trace("Creating remote ordering constraints"); apply_remote_node_ordering(data_set); crm_trace("Processing fencing and shutdown cases"); if (any_managed_resources(data_set) == FALSE) { crm_notice("Delaying fencing operations until there are resources to manage"); need_stonith = FALSE; } /* Check each node for stonith/shutdown */ for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; /* Guest nodes are "fenced" by recovering their container resource, * so handle them separately. */ if (is_container_remote_node(node)) { if (node->details->remote_requires_reset && need_stonith) { fence_guest(node, done, data_set); } continue; } stonith_op = NULL; if (node->details->unclean && need_stonith && pe_can_fence(data_set, node)) { stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set); pe_warn("Scheduling Node %s for STONITH", node->details->uname); stonith_constraints(node, stonith_op, data_set); if (node->details->is_dc) { dc_down = stonith_op; dc_fence = stonith_op; } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) { if (last_stonith) { order_actions(last_stonith, stonith_op, pe_order_optional); } last_stonith = stonith_op; } else { order_actions(stonith_op, done, pe_order_implies_then); stonith_ops = g_list_append(stonith_ops, stonith_op); } } else if (node->details->online && node->details->shutdown && /* TODO define what a shutdown op means for a remote node. * For now we do not send shutdown operations for remote nodes, but * if we can come up with a good use for this in the future, we will. */ is_remote_node(node) == FALSE) { action_t *down_op = NULL; crm_notice("Scheduling Node %s for shutdown", node->details->uname); down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname), CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set); shutdown_constraints(node, down_op, data_set); add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); if (node->details->is_dc) { dc_down = down_op; } } if (node->details->unclean && stonith_op == NULL) { integrity_lost = TRUE; pe_warn("Node %s is unclean!", node->details->uname); } } if (integrity_lost) { if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) { pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED"); pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE"); } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) { crm_notice("Cannot fence unclean nodes until quorum is" " attained (or no-quorum-policy is set to ignore)"); } } if (dc_down != NULL) { GListPtr gIter = NULL; crm_trace("Ordering shutdowns before %s on %s (DC)", dc_down->task, dc_down->node->details->uname); add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *node_stop = (action_t *) gIter->data; if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) { continue; } else if (node_stop->node->details->is_dc) { continue; } crm_debug("Ordering shutdown on %s before %s on %s", node_stop->node->details->uname, dc_down->task, dc_down->node->details->uname); order_actions(node_stop, dc_down, pe_order_optional); } if (last_stonith) { if (dc_down != last_stonith) { order_actions(last_stonith, dc_down, pe_order_optional); } } else { GListPtr gIter2 = NULL; for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) { stonith_op = (action_t *) gIter2->data; if (dc_down != stonith_op) { order_actions(stonith_op, dc_down, pe_order_optional); } } } } if (dc_fence) { order_actions(dc_down, done, pe_order_implies_then); } else if (last_stonith) { order_actions(last_stonith, done, pe_order_implies_then); } order_actions(done, all_stopped, pe_order_implies_then); g_list_free(stonith_ops); return TRUE; } /* * Determine the sets of independent actions and the correct order for the * actions in each set. * * Mark dependencies of un-runnable actions un-runnable * */ static GListPtr find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key) { GListPtr list = NULL; list = find_actions(actions, original_key, NULL); if (list == NULL) { /* we're potentially searching a child of the original resource */ char *key = NULL; char *tmp = NULL; char *task = NULL; int interval = 0; if (parse_op_key(original_key, &tmp, &task, &interval)) { key = generate_op_key(rsc->id, task, interval); /* crm_err("looking up %s instead of %s", key, original_key); */ /* slist_iter(action, action_t, actions, lpc, */ /* crm_err(" - %s", action->uuid)); */ list = find_actions(actions, key, NULL); } else { crm_err("search key: %s", original_key); } free(key); free(tmp); free(task); } return list; } static void rsc_order_then(action_t * lh_action, resource_t * rsc, order_constraint_t * order) { GListPtr gIter = NULL; GListPtr rh_actions = NULL; action_t *rh_action = NULL; enum pe_ordering type = order->type; CRM_CHECK(rsc != NULL, return); CRM_CHECK(order != NULL, return); rh_action = order->rh_action; crm_trace("Processing RH of ordering constraint %d", order->id); if (rh_action != NULL) { rh_actions = g_list_prepend(NULL, rh_action); } else if (rsc != NULL) { rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task); } if (rh_actions == NULL) { pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..." " ignoring", rsc->id, order->rh_action_task); if (lh_action) { pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid); } return; } if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) { pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid, order->rh_action_task); clear_bit(type, pe_order_implies_then); } gIter = rh_actions; for (; gIter != NULL; gIter = gIter->next) { action_t *rh_action_iter = (action_t *) gIter->data; if (lh_action) { order_actions(lh_action, rh_action_iter, type); } else if (type & pe_order_implies_then) { update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type); } else { crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type); } } g_list_free(rh_actions); } static void rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_t * data_set) { GListPtr gIter = NULL; GListPtr lh_actions = NULL; action_t *lh_action = order->lh_action; resource_t *rh_rsc = order->rh_rsc; crm_trace("Processing LH of ordering constraint %d", order->id); CRM_ASSERT(lh_rsc != NULL); if (lh_action != NULL) { lh_actions = g_list_prepend(NULL, lh_action); } else if (lh_action == NULL) { lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task); } if (lh_actions == NULL && lh_rsc != rh_rsc) { char *key = NULL; char *rsc_id = NULL; char *op_type = NULL; int interval = 0; parse_op_key(order->lh_action_task, &rsc_id, &op_type, &interval); key = generate_op_key(lh_rsc->id, op_type, interval); if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) { free(key); pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring", lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task); } else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) { free(key); pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring", lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task); } else { pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating", lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task); lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set); lh_actions = g_list_prepend(NULL, lh_action); } free(op_type); free(rsc_id); } gIter = lh_actions; for (; gIter != NULL; gIter = gIter->next) { action_t *lh_action_iter = (action_t *) gIter->data; if (rh_rsc == NULL && order->rh_action) { rh_rsc = order->rh_action->rsc; } if (rh_rsc) { rsc_order_then(lh_action_iter, rh_rsc, order); } else if (order->rh_action) { order_actions(lh_action_iter, order->rh_action, order->type); } } g_list_free(lh_actions); } extern gboolean update_action(action_t * action); extern void update_colo_start_chain(action_t * action); static int is_recurring_action(action_t *action) { const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL); int interval = crm_parse_int(interval_s, "0"); if(interval > 0) { return TRUE; } return FALSE; } static void apply_container_ordering(action_t *action, pe_working_set_t *data_set) { /* VMs are also classified as containers for these purposes... in * that they both involve a 'thing' running on a real or remote * cluster node. * * This allows us to be smarter about the type and extent of * recovery actions required in various scenarios */ resource_t *remote_rsc = NULL; resource_t *container = NULL; enum action_tasks task = text2task(action->task); if (action->rsc == NULL) { return; } CRM_ASSERT(action->node); CRM_ASSERT(is_remote_node(action->node)); remote_rsc = action->node->details->remote_rsc; CRM_ASSERT(remote_rsc); container = remote_rsc->container; CRM_ASSERT(container); if(is_set(container->flags, pe_rsc_failed)) { pe_fence_node(data_set, action->node, "container failed"); } crm_trace("Order %s action %s relative to %s%s for %s%s", action->task, action->uuid, is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "", remote_rsc->id, is_set(container->flags, pe_rsc_failed)? "failed " : "", container->id); switch (task) { case start_rsc: case action_promote: /* Force resource recovery if the container is recovered */ custom_action_order(container, generate_op_key(container->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_implies_then | pe_order_runnable_left, data_set); /* Wait for the connection resource to be up too */ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left, data_set); break; case stop_rsc: if(is_set(container->flags, pe_rsc_failed)) { /* When the container representing a guest node fails, * the stop action for all the resources living in * that container is implied by the container * stopping. This is similar to how fencing operations * work for cluster nodes. */ } else { /* Otherwise, ensure the operation happens before the connection is brought down */ custom_action_order(action->rsc, NULL, action, remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL, pe_order_preserve, data_set); } break; case action_demote: if(is_set(container->flags, pe_rsc_failed)) { /* Just like a stop, the demote is implied by the * container having failed/stopped * * If we really wanted to we would order the demote * after the stop, IFF the containers current role was * stopped (otherwise we re-introduce an ordering * loop) */ } else { /* Otherwise, ensure the operation happens before the connection is brought down */ custom_action_order(action->rsc, NULL, action, remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL, pe_order_preserve, data_set); } break; default: /* Wait for the connection resource to be up */ if (is_recurring_action(action)) { /* In case we ever get the recovery logic wrong, force * recurring monitors to be restarted, even if just * the connection was re-established */ if(task != no_action) { custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set); } } else { custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left, data_set); } break; } } static enum remote_connection_state get_remote_node_state(pe_node_t *node) { resource_t *remote_rsc = NULL; node_t *cluster_node = NULL; if(node == NULL) { return remote_state_unknown; } remote_rsc = node->details->remote_rsc; CRM_ASSERT(remote_rsc); if(remote_rsc->running_on) { cluster_node = remote_rsc->running_on->data; } /* If the cluster node the remote connection resource resides on * is unclean or went offline, we can't process any operations * on that remote node until after it starts elsewhere. */ if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) { /* There is nowhere left to run the connection resource, * and the resource is in a failed state (either directly * or because it is located on a failed node). * * If there are any resources known to be active on it (stop), * or if there are resources in an unknown state (probe), we * must assume the worst and fence it. */ if (is_set(remote_rsc->flags, pe_rsc_failed)) { return remote_state_failed; } else if(cluster_node && cluster_node->details->unclean) { return remote_state_failed; } else { return remote_state_stopped; } } else if (cluster_node == NULL) { /* Connection is recoverable but not currently running anywhere, see if we can recover it first */ return remote_state_unknown; } else if(cluster_node->details->unclean == TRUE || cluster_node->details->online == FALSE) { /* Connection is running on a dead node, see if we can recover it first */ return remote_state_resting; } else if (g_list_length(remote_rsc->running_on) > 1 && remote_rsc->partial_migration_source && remote_rsc->partial_migration_target) { /* We're in the middle of migrating a connection resource, * wait until after the resource migrates before performing * any actions. */ return remote_state_resting; } return remote_state_alive; } static void apply_remote_ordering(action_t *action, pe_working_set_t *data_set) { resource_t *remote_rsc = NULL; node_t *cluster_node = NULL; enum action_tasks task = text2task(action->task); enum remote_connection_state state = get_remote_node_state(action->node); enum pe_ordering order_opts = pe_order_none; if (action->rsc == NULL) { return; } CRM_ASSERT(action->node); CRM_ASSERT(is_remote_node(action->node)); remote_rsc = action->node->details->remote_rsc; CRM_ASSERT(remote_rsc); if(remote_rsc->running_on) { cluster_node = remote_rsc->running_on->data; } crm_trace("Order %s action %s relative to %s%s (state %d)", action->task, action->uuid, is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "", remote_rsc->id, state); switch (task) { case start_rsc: case action_promote: /* This as an internally generated constraint exempt from * user constraint prohibitions, and this action isn't runnable * if the connection start isn't runnable. */ order_opts = pe_order_preserve | pe_order_runnable_left; if (state == remote_state_failed) { /* Force recovery, by making this action required */ order_opts |= pe_order_implies_then; } /* Ensure connection is up before running this action */ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, order_opts, data_set); break; case stop_rsc: /* Handle special case with remote node where stop actions need to be * ordered after the connection resource starts somewhere else. */ if(state == remote_state_resting) { /* Wait for the connection resource to be up and assume everything is as we left it */ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left, data_set); } else { if(state == remote_state_failed) { /* We would only be here if the resource is * running on the remote node. Since we have no * way to stop it, it is necessary to fence the * node. */ pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); } custom_action_order(action->rsc, NULL, action, remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL, pe_order_preserve | pe_order_implies_first, data_set); } break; case action_demote: /* Only order this demote relative to the connection start if the * connection isn't being torn down. Otherwise, the demote would be * blocked because the connection start would not be allowed. */ if(state == remote_state_resting || state == remote_state_unknown) { custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve, data_set); } /* Otherwise we can rely on the stop ordering */ break; default: /* Wait for the connection resource to be up */ if (is_recurring_action(action)) { /* In case we ever get the recovery logic wrong, force * recurring monitors to be restarted, even if just * the connection was re-established */ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set); } else { if(task == monitor_rsc && state == remote_state_failed) { /* We would only be here if we do not know the * state of the resource on the remote node. * Since we have no way to find out, it is * necessary to fence the node. */ pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable"); } if(cluster_node && state == remote_state_stopped) { /* The connection is currently up, but is going * down permanently. * * Make sure we check services are actually * stopped _before_ we let the connection get * closed */ custom_action_order(action->rsc, NULL, action, remote_rsc, generate_op_key(remote_rsc->id, RSC_STOP, 0), NULL, pe_order_preserve | pe_order_runnable_left, data_set); } else { custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, action->rsc, NULL, action, pe_order_preserve | pe_order_runnable_left, data_set); } } break; } } static void apply_remote_node_ordering(pe_working_set_t *data_set) { if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) { return; } for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (action->rsc == NULL) { continue; } /* Special case. */ if (action->rsc && action->rsc->is_remote_node && safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) { /* If we are clearing the failcount of an actual remote node * connection resource, then make sure this happens before allowing * the connection to start if we are planning on starting the * connection during this transition. */ custom_action_order(action->rsc, NULL, action, action->rsc, generate_op_key(action->rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set); continue; } /* If the action occurs on a Pacemaker Remote node, create * ordering constraints that guarantee the action occurs while the node * is active (after start, before stop ... things like that). */ if (action->node == NULL || is_remote_node(action->node) == FALSE || action->node->details->remote_rsc == NULL || is_set(action->flags, pe_action_pseudo)) { crm_trace("Nothing required for %s on %s", action->uuid, action->node?action->node->details->uname:"NA"); } else if(action->node->details->remote_rsc->container) { crm_trace("Container ordering for %s", action->uuid); apply_container_ordering(action, data_set); } else { crm_trace("Remote ordering for %s", action->uuid); apply_remote_ordering(action, data_set); } } } static void order_probes(pe_working_set_t * data_set) { #if 0 GListPtr gIter = NULL; for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; /* Given "A then B", we would prefer to wait for A to be * started before probing B. * * If A was a filesystem on which the binaries and data for B * lived, it would have been useful if the author of B's agent * could assume that A is running before B.monitor will be * called. * * However we can't _only_ probe once A is running, otherwise * we'd not detect the state of B if A could not be started * for some reason. * * In practice however, we cannot even do an opportunistic * version of this because B may be moving: * * B.probe -> B.start * B.probe -> B.stop * B.stop -> B.start * A.stop -> A.start * A.start -> B.probe * * So far so good, but if we add the result of this code: * * B.stop -> A.stop * * Then we get a loop: * * B.probe -> B.stop -> A.stop -> A.start -> B.probe * * We could kill the 'B.probe -> B.stop' dependency, but that * could mean stopping B "too" soon, because B.start must wait * for the probes to complete. * * Another option is to allow it only if A is a non-unique * clone with clone-max == node-max (since we'll never be * moving it). However, we could still be stopping one * instance at the same time as starting another. * The complexity of checking for allowed conditions combined * with the ever narrowing usecase suggests that this code * should remain disabled until someone gets smarter. */ action_t *start = NULL; GListPtr actions = NULL; GListPtr probes = NULL; char *key = NULL; key = start_key(rsc); actions = find_actions(rsc->actions, key, NULL); free(key); if (actions) { start = actions->data; g_list_free(actions); } if(start == NULL) { crm_err("No start action for %s", rsc->id); continue; } key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0); probes = find_actions(rsc->actions, key, NULL); free(key); for (actions = start->actions_before; actions != NULL; actions = actions->next) { action_wrapper_t *before = (action_wrapper_t *) actions->data; GListPtr pIter = NULL; action_t *first = before->action; resource_t *first_rsc = first->rsc; if(first->required_runnable_before) { GListPtr clone_actions = NULL; for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) { before = (action_wrapper_t *) clone_actions->data; crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid); CRM_ASSERT(before->action->rsc); first_rsc = before->action->rsc; break; } } else if(safe_str_neq(first->task, RSC_START)) { crm_trace("Not a start op %s for %s", first->uuid, start->uuid); } if(first_rsc == NULL) { continue; } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) { crm_trace("Same parent %s for %s", first_rsc->id, start->uuid); continue; } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) { crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid); continue; } crm_err("Appplying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant); for (pIter = probes; pIter != NULL; pIter = pIter->next) { action_t *probe = (action_t *) pIter->data; crm_err("Ordering %s before %s", first->uuid, probe->uuid); order_actions(first, probe, pe_order_optional); } } } #endif } gboolean stage7(pe_working_set_t * data_set) { GListPtr gIter = NULL; crm_trace("Applying ordering constraints"); /* Don't ask me why, but apparently they need to be processed in * the order they were created in... go figure * * Also g_list_append() has horrendous performance characteristics * So we need to use g_list_prepend() and then reverse the list here */ data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints); for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) { order_constraint_t *order = (order_constraint_t *) gIter->data; resource_t *rsc = order->lh_rsc; crm_trace("Applying ordering constraint: %d", order->id); if (rsc != NULL) { crm_trace("rsc_action-to-*"); rsc_order_first(rsc, order, data_set); continue; } rsc = order->rh_rsc; if (rsc != NULL) { crm_trace("action-to-rsc_action"); rsc_order_then(order->lh_action, rsc, order); } else { crm_trace("action-to-action"); order_actions(order->lh_action, order->rh_action, order->type); } } for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; update_colo_start_chain(action); } crm_trace("Ordering probes"); order_probes(data_set); crm_trace("Updating %d actions", g_list_length(data_set->actions)); for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; update_action(action); } LogNodeActions(data_set, FALSE); for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; LogActions(rsc, data_set, FALSE); } return TRUE; } int transition_id = -1; /* * Create a dependency graph to send to the transitioner (via the CRMd) */ gboolean stage8(pe_working_set_t * data_set) { GListPtr gIter = NULL; const char *value = NULL; transition_id++; crm_trace("Creating transition graph %d.", transition_id); data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH); value = pe_pref(data_set->config_hash, "cluster-delay"); crm_xml_add(data_set->graph, "cluster-delay", value); value = pe_pref(data_set->config_hash, "stonith-timeout"); crm_xml_add(data_set->graph, "stonith-timeout", value); crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY"); if (is_set(data_set->flags, pe_flag_start_failure_fatal)) { crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY"); } else { crm_xml_add(data_set->graph, "failed-start-offset", "1"); } value = pe_pref(data_set->config_hash, "batch-limit"); crm_xml_add(data_set->graph, "batch-limit", value); crm_xml_add_int(data_set->graph, "transition_id", transition_id); value = pe_pref(data_set->config_hash, "migration-limit"); if (crm_int_helper(value, NULL) > 0) { crm_xml_add(data_set->graph, "migration-limit", value); } /* errors... slist_iter(action, action_t, action_list, lpc, if(action->optional == FALSE && action->runnable == FALSE) { print_action("Ignoring", action, TRUE); } ); */ gIter = data_set->resources; for (; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id); rsc->cmds->expand(rsc, data_set); } crm_log_xml_trace(data_set->graph, "created resource-driven action list"); /* pseudo action to distribute list of nodes with maintenance state update */ add_maintenance_update(data_set); /* catch any non-resource specific actions */ crm_trace("processing non-resource actions"); gIter = data_set->actions; for (; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (action->rsc && action->node && action->node->details->shutdown && is_not_set(action->rsc->flags, pe_rsc_maintenance) && is_not_set(action->flags, pe_action_optional) && is_not_set(action->flags, pe_action_runnable) && crm_str_eq(action->task, RSC_STOP, TRUE) ) { /* Eventually we should just ignore the 'fence' case * But for now it's the best way to detect (in CTS) when * CIB resource updates are being lost */ if (is_set(data_set->flags, pe_flag_have_quorum) || data_set->no_quorum_policy == no_quorum_ignore) { crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)", action->node->details->unclean ? "fence" : "shut down", action->node->details->uname, action->rsc->id, is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked", is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "", action->uuid); } } graph_element_from_action(action, data_set); } crm_log_xml_trace(data_set->graph, "created generic action list"); crm_trace("Created transition graph %d.", transition_id); return TRUE; } void LogNodeActions(pe_working_set_t * data_set, gboolean terminal) { GListPtr gIter = NULL; for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { char *node_name = NULL; char *task = NULL; action_t *action = (action_t *) gIter->data; if (action->rsc != NULL) { continue; } else if (is_set(action->flags, pe_action_optional)) { continue; } if (is_container_remote_node(action->node)) { node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id); } else if(action->node) { node_name = crm_strdup_printf("%s", action->node->details->uname); } if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) { task = strdup("Shutdown"); } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { const char *op = g_hash_table_lookup(action->meta, "stonith_action"); task = crm_strdup_printf("Fence (%s)", op); } if(task == NULL) { /* Nothing to report */ } else if(terminal && action->reason) { printf(" * %s %s '%s'\n", task, node_name, action->reason); } else if(terminal) { printf(" * %s %s\n", task, node_name); } else if(action->reason) { crm_notice(" * %s %s '%s'\n", task, node_name, action->reason); } else { crm_notice(" * %s %s\n", task, node_name); } free(node_name); free(task); } } void cleanup_alloc_calculations(pe_working_set_t * data_set) { if (data_set == NULL) { return; } crm_trace("deleting %d order cons: %p", g_list_length(data_set->ordering_constraints), data_set->ordering_constraints); pe_free_ordering(data_set->ordering_constraints); data_set->ordering_constraints = NULL; crm_trace("deleting %d node cons: %p", g_list_length(data_set->placement_constraints), data_set->placement_constraints); pe_free_rsc_to_node(data_set->placement_constraints); data_set->placement_constraints = NULL; crm_trace("deleting %d inter-resource cons: %p", g_list_length(data_set->colocation_constraints), data_set->colocation_constraints); g_list_free_full(data_set->colocation_constraints, free); data_set->colocation_constraints = NULL; crm_trace("deleting %d ticket deps: %p", g_list_length(data_set->ticket_constraints), data_set->ticket_constraints); g_list_free_full(data_set->ticket_constraints, free); data_set->ticket_constraints = NULL; cleanup_calculations(data_set); } diff --git a/pengine/native.c b/pengine/native.c index 37cf5415cc..2e40a4ca21 100644 --- a/pengine/native.c +++ b/pengine/native.c @@ -1,3359 +1,3422 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include /* #define DELETE_THEN_REFRESH 1 // The crmd will remove the resource from the CIB itself, making this redundant */ #define INFINITY_HACK (INFINITY * -100) #define VARIANT_NATIVE 1 #include gboolean update_action(action_t * then); void native_rsc_colocation_rh_must(resource_t * rsc_lh, gboolean update_lh, resource_t * rsc_rh, gboolean update_rh); void native_rsc_colocation_rh_mustnot(resource_t * rsc_lh, gboolean update_lh, resource_t * rsc_rh, gboolean update_rh); void Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set); void RecurringOp(resource_t * rsc, action_t * start, node_t * node, xmlNode * operation, pe_working_set_t * data_set); void Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set); void RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node, xmlNode * operation, pe_working_set_t * data_set); void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set); gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set); gboolean StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); gboolean StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); gboolean DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); gboolean PromoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); gboolean RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); gboolean NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set); /* *INDENT-OFF* */ enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = { /* Current State */ /* Next State: Unknown Stopped Started Slave Master */ /* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, }, /* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, }, /* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, }; gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t*,node_t*,gboolean,pe_working_set_t*) = { /* Current State */ /* Next State: Unknown Stopped Started Slave Master */ /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, }, /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, }, /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, }, /* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, }, /* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, }, }; /* *INDENT-ON* */ static action_t * get_first_named_action(resource_t * rsc, const char *action, gboolean only_valid, node_t * current); static gboolean native_choose_node(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) { /* 1. Sort by weight 2. color.chosen_node = the node (of those with the highest wieght) with the fewest resources 3. remove color.chosen_node from all other colors */ GListPtr nodes = NULL; node_t *chosen = NULL; int lpc = 0; int multiple = 0; int length = 0; gboolean result = FALSE; process_utilization(rsc, &prefer, data_set); length = g_hash_table_size(rsc->allowed_nodes); if (is_not_set(rsc->flags, pe_rsc_provisional)) { return rsc->allocated_to ? TRUE : FALSE; } if(rsc->allowed_nodes) { nodes = g_hash_table_get_values(rsc->allowed_nodes); nodes = g_list_sort_with_data(nodes, sort_node_weight, g_list_nth_data(rsc->running_on, 0)); } if (prefer) { node_t *best = g_list_nth_data(nodes, 0); chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id); if (chosen && chosen->weight >= 0 && chosen->weight >= best->weight /* Possible alternative: (chosen->weight >= INFINITY || best->weight < INFINITY) */ && can_run_resources(chosen)) { pe_rsc_trace(rsc, "Using preferred node %s for %s instead of choosing from %d candidates", chosen->details->uname, rsc->id, length); } else if (chosen && chosen->weight < 0) { pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable", chosen->details->uname, rsc->id); chosen = NULL; } else if (chosen && can_run_resources(chosen)) { pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable", chosen->details->uname, rsc->id); chosen = NULL; } else { pe_rsc_trace(rsc, "Preferred node %s for %s was unknown", prefer->details->uname, rsc->id); } } if (chosen == NULL && rsc->allowed_nodes) { chosen = g_list_nth_data(nodes, 0); pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates", chosen ? chosen->details->uname : "", rsc->id, length); if (chosen && chosen->weight > 0 && can_run_resources(chosen)) { node_t *running = g_list_nth_data(rsc->running_on, 0); if (running && can_run_resources(running) == FALSE) { pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources", rsc->id, running->details->uname); running = NULL; } for (lpc = 1; lpc < length && running; lpc++) { node_t *tmp = g_list_nth_data(nodes, lpc); if (tmp->weight == chosen->weight) { multiple++; if (tmp->details == running->details) { /* prefer the existing node if scores are equal */ chosen = tmp; } } } } } if (multiple > 1) { int log_level = LOG_INFO; static char score[33]; score2char_stack(chosen->weight, score, sizeof(score)); if (chosen->weight >= INFINITY) { log_level = LOG_WARNING; } do_crm_log(log_level, "%d nodes with equal score (%s) for" " running %s resources. Chose %s.", multiple, score, rsc->id, chosen->details->uname); } result = native_assign_node(rsc, nodes, chosen, FALSE); g_list_free(nodes); return result; } static int node_list_attr_score(GHashTable * list, const char *attr, const char *value) { GHashTableIter iter; node_t *node = NULL; int best_score = -INFINITY; const char *best_node = NULL; if (attr == NULL) { attr = "#" XML_ATTR_UNAME; } g_hash_table_iter_init(&iter, list); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { int weight = node->weight; if (can_run_resources(node) == FALSE) { weight = -INFINITY; } if (weight > best_score || best_node == NULL) { const char *tmp = node_attribute_raw(node, attr); if (safe_str_eq(value, tmp)) { best_score = weight; best_node = node->details->uname; } } } if (safe_str_neq(attr, "#" XML_ATTR_UNAME)) { crm_info("Best score for %s=%s was %s with %d", attr, value, best_node ? best_node : "", best_score); } return best_score; } static void node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, float factor, gboolean only_positive) { int score = 0; int new_score = 0; GHashTableIter iter; node_t *node = NULL; if (attr == NULL) { attr = "#" XML_ATTR_UNAME; } g_hash_table_iter_init(&iter, list1); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { CRM_LOG_ASSERT(node != NULL); if(node == NULL) { continue; }; score = node_list_attr_score(list2, attr, node_attribute_raw(node, attr)); new_score = merge_weights(factor * score, node->weight); if (factor < 0 && score < 0) { /* Negative preference for a node with a negative score * should not become a positive preference * * TODO - Decide if we want to filter only if weight == -INFINITY * */ crm_trace("%s: Filtering %d + %f*%d (factor * score)", node->details->uname, node->weight, factor, score); } else if (node->weight == INFINITY_HACK) { crm_trace("%s: Filtering %d + %f*%d (node < 0)", node->details->uname, node->weight, factor, score); } else if (only_positive && new_score < 0 && node->weight > 0) { node->weight = INFINITY_HACK; crm_trace("%s: Filtering %d + %f*%d (score > 0)", node->details->uname, node->weight, factor, score); } else if (only_positive && new_score < 0 && node->weight == 0) { crm_trace("%s: Filtering %d + %f*%d (score == 0)", node->details->uname, node->weight, factor, score); } else { crm_trace("%s: %d + %f*%d", node->details->uname, node->weight, factor, score); node->weight = new_score; } } } GHashTable * node_hash_dup(GHashTable * hash) { /* Hack! */ GListPtr list = g_hash_table_get_values(hash); GHashTable *result = node_hash_from_list(list); g_list_free(list); return result; } GHashTable * native_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr, float factor, enum pe_weights flags) { return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags); } GHashTable * rsc_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr, float factor, enum pe_weights flags) { GHashTable *work = NULL; int multiplier = 1; if (factor < 0) { multiplier = -1; } if (is_set(rsc->flags, pe_rsc_merging)) { pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id); return nodes; } set_bit(rsc->flags, pe_rsc_merging); if (is_set(flags, pe_weights_init)) { if (rsc->variant == pe_group && rsc->children) { GListPtr last = rsc->children; while (last->next != NULL) { last = last->next; } pe_rsc_trace(rsc, "Merging %s as a group %p %p", rsc->id, rsc->children, last); work = rsc_merge_weights(last->data, rhs, NULL, attr, factor, flags); } else { work = node_hash_dup(rsc->allowed_nodes); } clear_bit(flags, pe_weights_init); } else if (rsc->variant == pe_group && rsc->children) { GListPtr iter = rsc->children; pe_rsc_trace(rsc, "%s: Combining scores from %d children of %s", rhs, g_list_length(iter), rsc->id); work = node_hash_dup(nodes); for(iter = rsc->children; iter->next != NULL; iter = iter->next) { work = rsc_merge_weights(iter->data, rhs, work, attr, factor, flags); } } else { pe_rsc_trace(rsc, "%s: Combining scores from %s", rhs, rsc->id); work = node_hash_dup(nodes); node_hash_update(work, rsc->allowed_nodes, attr, factor, is_set(flags, pe_weights_positive)); } if (is_set(flags, pe_weights_rollback) && can_run_any(work) == FALSE) { pe_rsc_info(rsc, "%s: Rolling back scores from %s", rhs, rsc->id); g_hash_table_destroy(work); clear_bit(rsc->flags, pe_rsc_merging); return nodes; } if (can_run_any(work)) { GListPtr gIter = NULL; if (is_set(flags, pe_weights_forward)) { gIter = rsc->rsc_cons; crm_trace("Checking %d additional colocation constraints", g_list_length(gIter)); } else if(rsc->variant == pe_group && rsc->children) { GListPtr last = rsc->children; while (last->next != NULL) { last = last->next; } gIter = ((resource_t*)last->data)->rsc_cons_lhs; crm_trace("Checking %d additional optional group colocation constraints from %s", g_list_length(gIter), ((resource_t*)last->data)->id); } else { gIter = rsc->rsc_cons_lhs; crm_trace("Checking %d additional optional colocation constraints %s", g_list_length(gIter), rsc->id); } for (; gIter != NULL; gIter = gIter->next) { resource_t *other = NULL; rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data; if (is_set(flags, pe_weights_forward)) { other = constraint->rsc_rh; } else { other = constraint->rsc_lh; } pe_rsc_trace(rsc, "Applying %s (%s)", constraint->id, other->id); work = rsc_merge_weights(other, rhs, work, constraint->node_attribute, multiplier * (float)constraint->score / INFINITY, flags|pe_weights_rollback); dump_node_scores(LOG_TRACE, NULL, rhs, work); } } if (is_set(flags, pe_weights_positive)) { node_t *node = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, work); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (node->weight == INFINITY_HACK) { node->weight = 1; } } } if (nodes) { g_hash_table_destroy(nodes); } clear_bit(rsc->flags, pe_rsc_merging); return work; } node_t * native_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) { GListPtr gIter = NULL; int alloc_details = scores_log_level + 1; if (rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) { /* never allocate children on their own */ pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id, rsc->parent->id); rsc->parent->cmds->allocate(rsc->parent, prefer, data_set); } if (is_not_set(rsc->flags, pe_rsc_provisional)) { return rsc->allocated_to; } if (is_set(rsc->flags, pe_rsc_allocating)) { pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id); return NULL; } set_bit(rsc->flags, pe_rsc_allocating); print_resource(alloc_details, "Allocating: ", rsc, FALSE); dump_node_scores(alloc_details, rsc, "Pre-alloc", rsc->allowed_nodes); for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) { rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data; GHashTable *archive = NULL; resource_t *rsc_rh = constraint->rsc_rh; pe_rsc_trace(rsc, "%s: Pre-Processing %s (%s, %d, %s)", rsc->id, constraint->id, rsc_rh->id, constraint->score, role2text(constraint->role_lh)); if (constraint->role_lh >= RSC_ROLE_MASTER || (constraint->score < 0 && constraint->score > -INFINITY)) { archive = node_hash_dup(rsc->allowed_nodes); } rsc_rh->cmds->allocate(rsc_rh, NULL, data_set); rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint); if (archive && can_run_any(rsc->allowed_nodes) == FALSE) { pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id); g_hash_table_destroy(rsc->allowed_nodes); rsc->allowed_nodes = archive; archive = NULL; } if (archive) { g_hash_table_destroy(archive); } } dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes); for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) { rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data; rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes, constraint->node_attribute, (float)constraint->score / INFINITY, pe_weights_rollback); } print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE); if (rsc->next_role == RSC_ROLE_STOPPED) { pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id); /* make sure it doesn't come up again */ resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set); } else if(rsc->next_role > rsc->role && is_set(data_set->flags, pe_flag_have_quorum) == FALSE && data_set->no_quorum_policy == no_quorum_freeze) { crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze", rsc->id, role2text(rsc->role), role2text(rsc->next_role)); rsc->next_role = rsc->role; } dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes); if (is_set(data_set->flags, pe_flag_stonith_enabled) && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) { clear_bit(rsc->flags, pe_rsc_managed); } if (is_not_set(rsc->flags, pe_rsc_managed)) { const char *reason = NULL; node_t *assign_to = NULL; rsc->next_role = rsc->role; if (rsc->running_on == NULL) { reason = "inactive"; } else if (rsc->role == RSC_ROLE_MASTER) { assign_to = rsc->running_on->data; reason = "master"; } else if (is_set(rsc->flags, pe_rsc_failed)) { assign_to = rsc->running_on->data; reason = "failed"; } else { assign_to = rsc->running_on->data; reason = "active"; } pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id, assign_to ? assign_to->details->uname : "'nowhere'", reason); native_assign_node(rsc, NULL, assign_to, TRUE); } else if (is_set(data_set->flags, pe_flag_stop_everything)) { pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id); native_assign_node(rsc, NULL, NULL, TRUE); } else if (is_set(rsc->flags, pe_rsc_provisional) && native_choose_node(rsc, prefer, data_set)) { pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id, rsc->allocated_to->details->uname); } else if (rsc->allocated_to == NULL) { if (is_not_set(rsc->flags, pe_rsc_orphan)) { pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id); } else if (rsc->running_on != NULL) { pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id); } } else { pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id, rsc->allocated_to->details->uname); } clear_bit(rsc->flags, pe_rsc_allocating); print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE); if (rsc->is_remote_node) { node_t *remote_node = pe_find_node(data_set->nodes, rsc->id); CRM_ASSERT(remote_node != NULL); if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) { crm_trace("Setting remote node %s to ONLINE", remote_node->details->id); remote_node->details->online = TRUE; /* We shouldn't consider an unseen remote-node unclean if we are going * to try and connect to it. Otherwise we get an unnecessary fence */ if (remote_node->details->unseen == TRUE) { remote_node->details->unclean = FALSE; } } else { crm_trace("Setting remote node %s to SHUTDOWN. next role = %s, allocated=%s", remote_node->details->id, role2text(rsc->next_role), rsc->allocated_to ? "true" : "false"); remote_node->details->shutdown = TRUE; } } return rsc->allocated_to; } static gboolean is_op_dup(resource_t * rsc, const char *name, const char *interval) { gboolean dup = FALSE; const char *id = NULL; const char *value = NULL; xmlNode *operation = NULL; CRM_ASSERT(rsc); for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; operation = __xml_next_element(operation)) { if (crm_str_eq((const char *)operation->name, "op", TRUE)) { value = crm_element_value(operation, "name"); if (safe_str_neq(value, name)) { continue; } value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); if (value == NULL) { value = "0"; } if (safe_str_neq(value, interval)) { continue; } if (id == NULL) { id = ID(operation); } else { crm_config_err("Operation %s is a duplicate of %s", ID(operation), id); crm_config_err ("Do not use the same (name, interval) combination more than once per resource"); dup = TRUE; } } } return dup; } void RecurringOp(resource_t * rsc, action_t * start, node_t * node, xmlNode * operation, pe_working_set_t * data_set) { char *key = NULL; const char *name = NULL; const char *value = NULL; const char *interval = NULL; const char *node_uname = NULL; unsigned long long interval_ms = 0; action_t *mon = NULL; gboolean is_optional = TRUE; GListPtr possible_matches = NULL; /* Only process for the operations without role="Stopped" */ value = crm_element_value(operation, "role"); if (value && text2role(value) == RSC_ROLE_STOPPED) { return; } CRM_ASSERT(rsc); pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s", ID(operation), rsc->id, role2text(rsc->next_role), node ? node->details->uname : "n/a"); if (node != NULL) { node_uname = node->details->uname; } interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); interval_ms = crm_get_interval(interval); if (interval_ms == 0) { return; } name = crm_element_value(operation, "name"); if (is_op_dup(rsc, name, interval)) { return; } if (safe_str_eq(name, RSC_STOP) || safe_str_eq(name, RSC_START) || safe_str_eq(name, RSC_DEMOTE) || safe_str_eq(name, RSC_PROMOTE) ) { crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name); return; } key = generate_op_key(rsc->id, name, interval_ms); if (find_rsc_op_entry(rsc, key) == NULL) { /* disabled */ free(key); return; } if (start != NULL) { pe_rsc_trace(rsc, "Marking %s %s due to %s", key, is_set(start->flags, pe_action_optional) ? "optional" : "mandatory", start->uuid); is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional); } else { pe_rsc_trace(rsc, "Marking %s optional", key); is_optional = TRUE; } /* start a monitor for an already active resource */ possible_matches = find_actions_exact(rsc->actions, key, node); if (possible_matches == NULL) { is_optional = FALSE; pe_rsc_trace(rsc, "Marking %s mandatory: not active", key); } else { GListPtr gIter = NULL; for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) { action_t *op = (action_t *) gIter->data; if (is_set(op->flags, pe_action_reschedule)) { is_optional = FALSE; break; } } g_list_free(possible_matches); } if ((rsc->next_role == RSC_ROLE_MASTER && value == NULL) || (value != NULL && text2role(value) != rsc->next_role)) { int log_level = LOG_DEBUG_2; const char *result = "Ignoring"; if (is_optional) { char *local_key = strdup(key); log_level = LOG_INFO; result = "Cancelling"; /* it's running : cancel it */ mon = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set); free(mon->task); free(mon->cancel_task); mon->task = strdup(RSC_CANCEL); mon->cancel_task = strdup(name); add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval); add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name); local_key = NULL; switch (rsc->role) { case RSC_ROLE_SLAVE: case RSC_ROLE_STARTED: if (rsc->next_role == RSC_ROLE_MASTER) { local_key = promote_key(rsc); } else if (rsc->next_role == RSC_ROLE_STOPPED) { local_key = stop_key(rsc); } break; case RSC_ROLE_MASTER: local_key = demote_key(rsc); break; default: break; } if (local_key) { custom_action_order(rsc, NULL, mon, rsc, local_key, NULL, pe_order_runnable_left, data_set); } mon = NULL; } do_crm_log(log_level, "%s action %s (%s vs. %s)", result, key, value ? value : role2text(RSC_ROLE_SLAVE), role2text(rsc->next_role)); free(key); return; } mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set); key = mon->uuid; if (is_optional) { pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(node_uname), mon->uuid); } if (start == NULL || is_set(start->flags, pe_action_runnable) == FALSE) { pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)", crm_str(node_uname), mon->uuid); update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } else if (node == NULL || node->details->online == FALSE || node->details->unclean) { pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)", crm_str(node_uname), mon->uuid); update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } else if (is_set(mon->flags, pe_action_optional) == FALSE) { pe_rsc_info(rsc, " Start recurring %s (%llus) for %s on %s", mon->task, interval_ms / 1000, rsc->id, crm_str(node_uname)); } if (rsc->next_role == RSC_ROLE_MASTER) { char *running_master = crm_itoa(PCMK_OCF_RUNNING_MASTER); add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master); free(running_master); } if (node == NULL || is_set(rsc->flags, pe_rsc_managed)) { custom_action_order(rsc, start_key(rsc), NULL, NULL, strdup(key), mon, pe_order_implies_then | pe_order_runnable_left, data_set); custom_action_order(rsc, reload_key(rsc), NULL, NULL, strdup(key), mon, pe_order_implies_then | pe_order_runnable_left, data_set); if (rsc->next_role == RSC_ROLE_MASTER) { custom_action_order(rsc, promote_key(rsc), NULL, rsc, NULL, mon, pe_order_optional | pe_order_runnable_left, data_set); } else if (rsc->role == RSC_ROLE_MASTER) { custom_action_order(rsc, demote_key(rsc), NULL, rsc, NULL, mon, pe_order_optional | pe_order_runnable_left, data_set); } } } void Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set) { if (is_not_set(rsc->flags, pe_rsc_maintenance) && (node == NULL || node->details->maintenance == FALSE)) { xmlNode *operation = NULL; for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; operation = __xml_next_element(operation)) { if (crm_str_eq((const char *)operation->name, "op", TRUE)) { RecurringOp(rsc, start, node, operation, data_set); } } } } void RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node, xmlNode * operation, pe_working_set_t * data_set) { char *key = NULL; const char *name = NULL; const char *role = NULL; const char *interval = NULL; const char *node_uname = NULL; unsigned long long interval_ms = 0; GListPtr possible_matches = NULL; GListPtr gIter = NULL; /* TODO: Support of non-unique clone */ if (is_set(rsc->flags, pe_rsc_unique) == FALSE) { return; } /* Only process for the operations with role="Stopped" */ role = crm_element_value(operation, "role"); if (role == NULL || text2role(role) != RSC_ROLE_STOPPED) { return; } pe_rsc_trace(rsc, "Creating recurring actions %s for %s in role %s on nodes where it'll not be running", ID(operation), rsc->id, role2text(rsc->next_role)); if (node != NULL) { node_uname = node->details->uname; } interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); interval_ms = crm_get_interval(interval); if (interval_ms == 0) { return; } name = crm_element_value(operation, "name"); if (is_op_dup(rsc, name, interval)) { return; } if (safe_str_eq(name, RSC_STOP) || safe_str_eq(name, RSC_START) || safe_str_eq(name, RSC_DEMOTE) || safe_str_eq(name, RSC_PROMOTE) ) { crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name); return; } key = generate_op_key(rsc->id, name, interval_ms); if (find_rsc_op_entry(rsc, key) == NULL) { /* disabled */ free(key); return; } /* if the monitor exists on the node where the resource will be running, cancel it */ if (node != NULL) { possible_matches = find_actions_exact(rsc->actions, key, node); if (possible_matches) { action_t *cancel_op = NULL; char *local_key = strdup(key); g_list_free(possible_matches); cancel_op = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set); free(cancel_op->task); free(cancel_op->cancel_task); cancel_op->task = strdup(RSC_CANCEL); cancel_op->cancel_task = strdup(name); add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL, interval); add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, name); local_key = NULL; if (rsc->next_role == RSC_ROLE_STARTED || rsc->next_role == RSC_ROLE_SLAVE) { /* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */ /* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */ custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL, pe_order_runnable_left, data_set); } pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s", key, role, role2text(rsc->next_role), crm_str(node_uname)); } } for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *stop_node = (node_t *) gIter->data; const char *stop_node_uname = stop_node->details->uname; gboolean is_optional = TRUE; gboolean probe_is_optional = TRUE; gboolean stop_is_optional = TRUE; action_t *stopped_mon = NULL; char *rc_inactive = NULL; GListPtr probe_complete_ops = NULL; GListPtr stop_ops = NULL; GListPtr local_gIter = NULL; char *stop_op_key = NULL; if (node_uname && safe_str_eq(stop_node_uname, node_uname)) { continue; } pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s", ID(operation), rsc->id, crm_str(stop_node_uname)); /* start a monitor for an already stopped resource */ possible_matches = find_actions_exact(rsc->actions, key, stop_node); if (possible_matches == NULL) { pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key, crm_str(stop_node_uname)); is_optional = FALSE; } else { pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key, crm_str(stop_node_uname)); is_optional = TRUE; g_list_free(possible_matches); } stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set); rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING); add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive); free(rc_inactive); if (is_set(rsc->flags, pe_rsc_managed)) { char *probe_key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0); GListPtr probes = find_actions(rsc->actions, probe_key, stop_node); GListPtr pIter = NULL; for (pIter = probes; pIter != NULL; pIter = pIter->next) { action_t *probe = (action_t *) pIter->data; order_actions(probe, stopped_mon, pe_order_runnable_left); crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname); } g_list_free(probes); free(probe_key); } if (probe_complete_ops) { g_list_free(probe_complete_ops); } stop_op_key = stop_key(rsc); stop_ops = find_actions_exact(rsc->actions, stop_op_key, stop_node); for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) { action_t *stop = (action_t *) local_gIter->data; if (is_set(stop->flags, pe_action_optional) == FALSE) { stop_is_optional = FALSE; } if (is_set(stop->flags, pe_action_runnable) == FALSE) { crm_debug("%s\t %s (cancelled : stop un-runnable)", crm_str(stop_node_uname), stopped_mon->uuid); update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } if (is_set(rsc->flags, pe_rsc_managed)) { custom_action_order(rsc, strdup(stop_op_key), stop, NULL, strdup(key), stopped_mon, pe_order_implies_then | pe_order_runnable_left, data_set); } } if (stop_ops) { g_list_free(stop_ops); } free(stop_op_key); if (is_optional == FALSE && probe_is_optional && stop_is_optional && is_set(rsc->flags, pe_rsc_managed) == FALSE) { pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged", key, crm_str(stop_node_uname)); update_action_flags(stopped_mon, pe_action_optional, __FUNCTION__, __LINE__); } if (is_set(stopped_mon->flags, pe_action_optional)) { pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid); } if (stop_node->details->online == FALSE || stop_node->details->unclean) { pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)", crm_str(stop_node_uname), stopped_mon->uuid); update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } if (is_set(stopped_mon->flags, pe_action_runnable) && is_set(stopped_mon->flags, pe_action_optional) == FALSE) { crm_notice(" Start recurring %s (%llus) for %s on %s", stopped_mon->task, interval_ms / 1000, rsc->id, crm_str(stop_node_uname)); } } free(key); } void Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set) { if (is_not_set(rsc->flags, pe_rsc_maintenance) && (node == NULL || node->details->maintenance == FALSE)) { xmlNode *operation = NULL; for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; operation = __xml_next_element(operation)) { if (crm_str_eq((const char *)operation->name, "op", TRUE)) { RecurringOp_Stopped(rsc, start, node, operation, data_set); } } } } static void handle_migration_actions(resource_t * rsc, node_t *current, node_t *chosen, pe_working_set_t * data_set) { action_t *migrate_to = NULL; action_t *migrate_from = NULL; action_t *start = NULL; action_t *stop = NULL; gboolean partial = rsc->partial_migration_target ? TRUE : FALSE; pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s", rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE"); start = start_action(rsc, chosen, TRUE); stop = stop_action(rsc, current, TRUE); if (partial == FALSE) { migrate_to = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), RSC_MIGRATE, current, TRUE, TRUE, data_set); } migrate_from = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), RSC_MIGRATED, chosen, TRUE, TRUE, data_set); if ((migrate_to && migrate_from) || (migrate_from && partial)) { set_bit(start->flags, pe_action_migrate_runnable); set_bit(stop->flags, pe_action_migrate_runnable); update_action_flags(start, pe_action_pseudo, __FUNCTION__, __LINE__); /* easier than trying to delete it from the graph */ /* order probes before migrations */ if (partial) { set_bit(migrate_from->flags, pe_action_migrate_runnable); migrate_from->needs = start->needs; custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL, rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional, data_set); } else { set_bit(migrate_from->flags, pe_action_migrate_runnable); set_bit(migrate_to->flags, pe_action_migrate_runnable); migrate_to->needs = start->needs; custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL, rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL, pe_order_optional, data_set); custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL, rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional | pe_order_implies_first_migratable, data_set); } custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_optional | pe_order_implies_first_migratable, data_set); custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional | pe_order_implies_first_migratable | pe_order_pseudo_left, data_set); } if (migrate_to) { add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname); add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname); /* pcmk remote connections don't require pending to be recorded in cib. * We can optimize cib writes by only setting PENDING for non pcmk remote * connection resources */ if (rsc->is_remote_node == FALSE) { /* migrate_to takes place on the source node, but can * have an effect on the target node depending on how * the agent is written. Because of this, we have to maintain * a record that the migrate_to occurred incase the source node * loses membership while the migrate_to action is still in-flight. */ add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true"); } } if (migrate_from) { add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname); add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname); } } void native_create_actions(resource_t * rsc, pe_working_set_t * data_set) { action_t *start = NULL; node_t *chosen = NULL; node_t *current = NULL; gboolean need_stop = FALSE; gboolean is_moving = FALSE; gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE; GListPtr gIter = NULL; int num_active_nodes = 0; enum rsc_role_e role = RSC_ROLE_UNKNOWN; enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; CRM_ASSERT(rsc); chosen = rsc->allocated_to; if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) { rsc->next_role = RSC_ROLE_STARTED; pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role)); } else if (rsc->next_role == RSC_ROLE_UNKNOWN) { rsc->next_role = RSC_ROLE_STOPPED; pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role)); } pe_rsc_trace(rsc, "Processing state transition for %s %p: %s->%s", rsc->id, rsc, role2text(rsc->role), role2text(rsc->next_role)); if (rsc->running_on) { current = rsc->running_on->data; } for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { node_t *n = (node_t *) gIter->data; if (rsc->partial_migration_source && (n->details == rsc->partial_migration_source->details)) { current = rsc->partial_migration_source; } num_active_nodes++; } for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) { node_t *current = (node_t *) gIter->data; action_t *stop = stop_action(rsc, current, FALSE); set_bit(stop->flags, pe_action_dangle); pe_rsc_trace(rsc, "Forcing a cleanup of %s on %s", rsc->id, current->details->uname); if (is_set(data_set->flags, pe_flag_remove_after_stop)) { DeleteRsc(rsc, current, FALSE, data_set); } } if (num_active_nodes > 1) { if (num_active_nodes == 2 && chosen && rsc->partial_migration_target && rsc->partial_migration_source && (current->details == rsc->partial_migration_source->details) && (chosen->details == rsc->partial_migration_target->details)) { /* Here the chosen node is still the migration target from a partial * migration. Attempt to continue the migration instead of recovering * by stopping the resource everywhere and starting it on a single node. */ pe_rsc_trace(rsc, "Will attempt to continue with a partial migration to target %s from %s", rsc->partial_migration_target->details->id, rsc->partial_migration_source->details->id); } else { const char *type = crm_element_value(rsc->xml, XML_ATTR_TYPE); const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); if(rsc->partial_migration_target && rsc->partial_migration_source) { crm_notice("Resource %s can no longer migrate to %s. Stopping on %s too", rsc->id, rsc->partial_migration_target->details->uname, rsc->partial_migration_source->details->uname); } else { pe_proc_err("Resource %s (%s::%s) is active on %d nodes %s", rsc->id, class, type, num_active_nodes, recovery2text(rsc->recovery_type)); crm_warn("See %s for more information.", "http://clusterlabs.org/wiki/FAQ#Resource_is_Too_Active"); } if (rsc->recovery_type == recovery_stop_start) { need_stop = TRUE; } /* If by chance a partial migration is in process, * but the migration target is not chosen still, clear all * partial migration data. */ rsc->partial_migration_source = rsc->partial_migration_target = NULL; allow_migrate = FALSE; } } if (is_set(rsc->flags, pe_rsc_start_pending)) { start = start_action(rsc, chosen, TRUE); set_bit(start->flags, pe_action_print_always); } if (current && chosen && current->details != chosen->details) { pe_rsc_trace(rsc, "Moving %s", rsc->id); is_moving = TRUE; need_stop = TRUE; } else if (is_set(rsc->flags, pe_rsc_failed)) { pe_rsc_trace(rsc, "Recovering %s", rsc->id); need_stop = TRUE; } else if (is_set(rsc->flags, pe_rsc_block)) { pe_rsc_trace(rsc, "Block %s", rsc->id); need_stop = TRUE; } else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) { /* Recovery of a promoted resource */ start = start_action(rsc, chosen, TRUE); if (is_set(start->flags, pe_action_optional) == FALSE) { pe_rsc_trace(rsc, "Forced start %s", rsc->id); need_stop = TRUE; } } pe_rsc_trace(rsc, "Creating actions for %s: %s->%s", rsc->id, role2text(rsc->role), role2text(rsc->next_role)); /* Create any additional actions required when bringing resource down and * back up to same level. */ role = rsc->role; while (role != RSC_ROLE_STOPPED) { next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED]; pe_rsc_trace(rsc, "Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role), rsc->id, need_stop ? " required" : ""); if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) { break; } role = next_role; } while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) { next_role = rsc_state_matrix[role][rsc->role]; pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role), rsc->id, need_stop ? " required" : ""); if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) { break; } role = next_role; } role = rsc->role; /* Required steps from this role to the next */ while (role != rsc->next_role) { next_role = rsc_state_matrix[role][rsc->next_role]; pe_rsc_trace(rsc, "Role: Executing: %s->%s = (%s on %s)", role2text(role), role2text(next_role), rsc->id, chosen?chosen->details->uname:"NA"); if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) { break; } role = next_role; } if(is_set(rsc->flags, pe_rsc_block)) { pe_rsc_trace(rsc, "No monitor additional ops for blocked resource"); } else if (rsc->next_role != RSC_ROLE_STOPPED || is_set(rsc->flags, pe_rsc_managed) == FALSE) { pe_rsc_trace(rsc, "Monitor ops for active resource"); start = start_action(rsc, chosen, TRUE); Recurring(rsc, start, chosen, data_set); Recurring_Stopped(rsc, start, chosen, data_set); } else { pe_rsc_trace(rsc, "Monitor ops for in-active resource"); Recurring_Stopped(rsc, NULL, NULL, data_set); } /* if we are stuck in a partial migration, where the target * of the partial migration no longer matches the chosen target. * A full stop/start is required */ if (rsc->partial_migration_target && (chosen == NULL || rsc->partial_migration_target->details != chosen->details)) { pe_rsc_trace(rsc, "Not allowing partial migration to continue. %s", rsc->id); allow_migrate = FALSE; } else if (is_moving == FALSE || is_not_set(rsc->flags, pe_rsc_managed) || is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending) || (current->details->unclean == TRUE) || rsc->next_role < RSC_ROLE_STARTED) { allow_migrate = FALSE; } if (allow_migrate) { handle_migration_actions(rsc, current, chosen, data_set); } } static void rsc_avoids_remote_nodes(resource_t *rsc) { GHashTableIter iter; node_t *node = NULL; g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (node->details->remote_rsc) { node->weight = -INFINITY; } } } void native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) { /* This function is on the critical path and worth optimizing as much as possible */ resource_t *top = uber_parent(rsc); int type = pe_order_optional | pe_order_implies_then | pe_order_restart; gboolean is_stonith = is_set(rsc->flags, pe_rsc_fence_device); custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, type, data_set); if (top->variant == pe_master || rsc->role > RSC_ROLE_SLAVE) { custom_action_order(rsc, generate_op_key(rsc->id, RSC_DEMOTE, 0), NULL, rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_implies_first_master, data_set); custom_action_order(rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, rsc, generate_op_key(rsc->id, RSC_PROMOTE, 0), NULL, pe_order_runnable_left, data_set); } if (is_stonith == FALSE && is_set(data_set->flags, pe_flag_enable_unfencing) && is_set(rsc->flags, pe_rsc_needs_unfencing)) { /* Check if the node needs to be unfenced first */ node_t *node = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); crm_debug("Ordering any stops of %s before %s, and any starts after", rsc->id, unfence->uuid); /* * It would be more efficient to order clone resources once, * rather than order each instance, but ordering the instance * allows us to avoid unnecessary dependencies that might conflict * with user constraints. * * @TODO: This constraint can still produce a transition loop if the * resource has a stop scheduled on the node being unfenced, and * there is a user ordering constraint to start some other resource * (which will be ordered after the unfence) before stopping this * resource. An example is "start some slow-starting cloned service * before stopping an associated virtual IP that may be moving to * it": * stop this -> unfencing -> start that -> stop this */ custom_action_order(rsc, stop_key(rsc), NULL, NULL, strdup(unfence->uuid), unfence, pe_order_optional|pe_order_same_node, data_set); custom_action_order(NULL, strdup(unfence->uuid), unfence, rsc, start_key(rsc), NULL, pe_order_implies_then_on_node|pe_order_same_node, data_set); } } if (is_not_set(rsc->flags, pe_rsc_managed)) { pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id); return; } { action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set); custom_action_order(rsc, stop_key(rsc), NULL, NULL, strdup(all_stopped->task), all_stopped, pe_order_implies_then | pe_order_runnable_left, data_set); } if (g_hash_table_size(rsc->utilization) > 0 && safe_str_neq(data_set->placement_strategy, "default")) { GHashTableIter iter; node_t *next = NULL; GListPtr gIter = NULL; pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s", rsc->id, data_set->placement_strategy); for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { node_t *current = (node_t *) gIter->data; char *load_stopped_task = crm_concat(LOAD_STOPPED, current->details->uname, '_'); action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set); if (load_stopped->node == NULL) { load_stopped->node = node_copy(current); update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); } custom_action_order(rsc, stop_key(rsc), NULL, NULL, load_stopped_task, load_stopped, pe_order_load, data_set); } g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&next)) { char *load_stopped_task = crm_concat(LOAD_STOPPED, next->details->uname, '_'); action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set); if (load_stopped->node == NULL) { load_stopped->node = node_copy(next); update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); } custom_action_order(NULL, strdup(load_stopped_task), load_stopped, rsc, start_key(rsc), NULL, pe_order_load, data_set); custom_action_order(NULL, strdup(load_stopped_task), load_stopped, rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL, pe_order_load, data_set); free(load_stopped_task); } } if (rsc->container) { resource_t *remote_rsc = NULL; /* A user can specify that a resource must start on a Pacemaker Remote * node by explicitly configuring it with the container=NODENAME * meta-attribute. This is of questionable merit, since location * constraints can accomplish the same thing. But we support it, so here * we check whether a resource (that is not itself a remote connection) * has container set to a remote node or guest node resource. */ if (rsc->container->is_remote_node) { remote_rsc = rsc->container; } else if (rsc->is_remote_node == FALSE) { remote_rsc = rsc_contains_remote_node(data_set, rsc->container); } if (remote_rsc) { /* The container represents a Pacemaker Remote node, so force the * resource on the Pacemaker Remote node instead of colocating the * resource with the container resource. */ GHashTableIter iter; node_t *node = NULL; g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if (node->details->remote_rsc != remote_rsc) { node->weight = -INFINITY; } } } else { /* This resource is either a filler for a container that does NOT * represent a Pacemaker Remote node, or a Pacemaker Remote * connection resource for a guest node or bundle. */ int score; crm_trace("Order and colocate %s relative to its container %s", rsc->id, rsc->container->id); custom_action_order(rsc->container, generate_op_key(rsc->container->id, RSC_START, 0), NULL, rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_implies_then | pe_order_runnable_left, data_set); custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, rsc->container, generate_op_key(rsc->container->id, RSC_STOP, 0), NULL, pe_order_implies_first, data_set); if (is_set(rsc->flags, pe_rsc_allow_remote_remotes)) { score = 10000; /* Highly preferred but not essential */ } else { score = INFINITY; /* Force them to run on the same host */ } rsc_colocation_new("resource-with-container", NULL, score, rsc, rsc->container, NULL, NULL, data_set); } } if (rsc->is_remote_node || is_stonith) { /* don't allow remote nodes to run stonith devices * or remote connection resources.*/ rsc_avoids_remote_nodes(rsc); } /* If this is a guest node's implicit remote connection, do not allow the * guest resource to live on a Pacemaker Remote node, to avoid nesting * remotes. However, allow bundles to run on remote nodes. */ if (rsc->is_remote_node && rsc->container && is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) { rsc_avoids_remote_nodes(rsc->container); } } void native_rsc_colocation_lh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint) { if (rsc_lh == NULL) { pe_err("rsc_lh was NULL for %s", constraint->id); return; } else if (constraint->rsc_rh == NULL) { pe_err("rsc_rh was NULL for %s", constraint->id); return; } pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id, rsc_rh->id); rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint); } enum filter_colocation_res filter_colocation_constraint(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint, gboolean preview) { if (constraint->score == 0) { return influence_nothing; } /* rh side must be allocated before we can process constraint */ if (preview == FALSE && is_set(rsc_rh->flags, pe_rsc_provisional)) { return influence_nothing; } if ((constraint->role_lh >= RSC_ROLE_SLAVE) && rsc_lh->parent && rsc_lh->parent->variant == pe_master && is_not_set(rsc_lh->flags, pe_rsc_provisional)) { /* LH and RH resources have already been allocated, place the correct * priority oh LH rsc for the given multistate resource role */ return influence_rsc_priority; } if (preview == FALSE && is_not_set(rsc_lh->flags, pe_rsc_provisional)) { /* error check */ struct node_shared_s *details_lh; struct node_shared_s *details_rh; if ((constraint->score > -INFINITY) && (constraint->score < INFINITY)) { return influence_nothing; } details_rh = rsc_rh->allocated_to ? rsc_rh->allocated_to->details : NULL; details_lh = rsc_lh->allocated_to ? rsc_lh->allocated_to->details : NULL; if (constraint->score == INFINITY && details_lh != details_rh) { crm_err("%s and %s are both allocated" " but to different nodes: %s vs. %s", rsc_lh->id, rsc_rh->id, details_lh ? details_lh->uname : "n/a", details_rh ? details_rh->uname : "n/a"); } else if (constraint->score == -INFINITY && details_lh == details_rh) { crm_err("%s and %s are both allocated" " but to the SAME node: %s", rsc_lh->id, rsc_rh->id, details_rh ? details_rh->uname : "n/a"); } return influence_nothing; } if (constraint->score > 0 && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) { crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s", role2text(constraint->role_lh), role2text(rsc_lh->next_role)); return influence_nothing; } if (constraint->score > 0 && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) { crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh)); return FALSE; } if (constraint->score < 0 && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) { crm_trace("LH: Skipping -ve constraint: \"%s\" state filter", role2text(constraint->role_lh)); return influence_nothing; } if (constraint->score < 0 && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) { crm_trace("RH: Skipping -ve constraint: \"%s\" state filter", role2text(constraint->role_rh)); return influence_nothing; } return influence_rsc_location; } static void influence_priority(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint) { const char *rh_value = NULL; const char *lh_value = NULL; const char *attribute = "#id"; int score_multiplier = 1; if (constraint->node_attribute != NULL) { attribute = constraint->node_attribute; } if (!rsc_rh->allocated_to || !rsc_lh->allocated_to) { return; } lh_value = node_attribute_raw(rsc_lh->allocated_to, attribute); rh_value = node_attribute_raw(rsc_rh->allocated_to, attribute); if (!safe_str_eq(lh_value, rh_value)) { if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) { rsc_lh->priority = -INFINITY; } return; } if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) { return; } if (constraint->role_lh == RSC_ROLE_SLAVE) { score_multiplier = -1; } rsc_lh->priority = merge_weights(score_multiplier * constraint->score, rsc_lh->priority); } static void colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint) { const char *tmp = NULL; const char *value = NULL; const char *attribute = "#id"; GHashTable *work = NULL; gboolean do_check = FALSE; GHashTableIter iter; node_t *node = NULL; if (constraint->node_attribute != NULL) { attribute = constraint->node_attribute; } if (rsc_rh->allocated_to) { value = node_attribute_raw(rsc_rh->allocated_to, attribute); do_check = TRUE; } else if (constraint->score < 0) { /* nothing to do: * anti-colocation with something that is not running */ return; } work = node_hash_dup(rsc_lh->allowed_nodes); g_hash_table_iter_init(&iter, work); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { tmp = node_attribute_raw(node, attribute); if (do_check && safe_str_eq(tmp, value)) { if (constraint->score < INFINITY) { pe_rsc_trace(rsc_lh, "%s: %s.%s += %d", constraint->id, rsc_lh->id, node->details->uname, constraint->score); node->weight = merge_weights(constraint->score, node->weight); } } else if (do_check == FALSE || constraint->score >= INFINITY) { pe_rsc_trace(rsc_lh, "%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id, node->details->uname, constraint->score, do_check ? "failed" : "unallocated"); node->weight = merge_weights(-constraint->score, node->weight); } } if (can_run_any(work) || constraint->score <= -INFINITY || constraint->score >= INFINITY) { g_hash_table_destroy(rsc_lh->allowed_nodes); rsc_lh->allowed_nodes = work; work = NULL; } else { static char score[33]; score2char_stack(constraint->score, score, sizeof(score)); pe_rsc_info(rsc_lh, "%s: Rolling back scores from %s (%d, %s)", rsc_lh->id, rsc_rh->id, do_check, score); } if (work) { g_hash_table_destroy(work); } } void native_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint) { enum filter_colocation_res filter_results; CRM_ASSERT(rsc_lh); CRM_ASSERT(rsc_rh); filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE); pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d, filter=%d)", constraint->score >= 0 ? "" : "Anti-", rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results); switch (filter_results) { case influence_rsc_priority: influence_priority(rsc_lh, rsc_rh, constraint); break; case influence_rsc_location: pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d)", constraint->score >= 0 ? "" : "Anti-", rsc_lh->id, rsc_rh->id, constraint->id, constraint->score); colocation_match(rsc_lh, rsc_rh, constraint); break; case influence_nothing: default: return; } } static gboolean filter_rsc_ticket(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket) { if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) { pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter", role2text(rsc_ticket->role_lh)); return FALSE; } return TRUE; } void rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set) { if (rsc_ticket == NULL) { pe_err("rsc_ticket was NULL"); return; } if (rsc_lh == NULL) { pe_err("rsc_lh was NULL for %s", rsc_ticket->id); return; } if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) { return; } if (rsc_lh->children) { GListPtr gIter = rsc_lh->children; pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id); for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; rsc_ticket_constraint(child_rsc, rsc_ticket, data_set); } return; } pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)", rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id, role2text(rsc_ticket->role_lh)); if (rsc_ticket->ticket->granted == FALSE && g_list_length(rsc_lh->running_on) > 0) { GListPtr gIter = NULL; switch (rsc_ticket->loss_policy) { case loss_ticket_stop: resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set); break; case loss_ticket_demote: /*Promotion score will be set to -INFINITY in master_promotion_order() */ if (rsc_ticket->role_lh != RSC_ROLE_MASTER) { resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set); } break; case loss_ticket_fence: if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) { return; } resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set); for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; pe_fence_node(data_set, node, "deadman ticket was lost"); } break; case loss_ticket_freeze: if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) { return; } if (g_list_length(rsc_lh->running_on) > 0) { clear_bit(rsc_lh->flags, pe_rsc_managed); set_bit(rsc_lh->flags, pe_rsc_block); } break; } } else if (rsc_ticket->ticket->granted == FALSE) { if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) { resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set); } } else if (rsc_ticket->ticket->standby) { if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) { resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set); } } } enum pe_action_flags native_action_flags(action_t * action, node_t * node) { return action->flags; } enum pe_graph_flags native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, enum pe_action_flags filter, enum pe_ordering type) { /* flags == get_action_flags(first, then_node) called from update_action() */ enum pe_graph_flags changed = pe_graph_none; enum pe_action_flags then_flags = then->flags; enum pe_action_flags first_flags = first->flags; crm_trace( "Testing %s on %s (0x%.6x) with %s 0x%.6x", first->uuid, first->node ? first->node->details->uname : "[none]", first->flags, then->uuid, then->flags); if (type & pe_order_asymmetrical) { resource_t *then_rsc = then->rsc; enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0; if (!then_rsc) { /* ignore */ } else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) { /* ignore... if 'then' is supposed to be stopped after 'first', but * then is already stopped, there is nothing to be done when non-symmetrical. */ } else if ((then_rsc_role >= RSC_ROLE_STARTED) && safe_str_eq(then->task, RSC_START) && then->node && then_rsc->running_on && g_list_length(then_rsc->running_on) == 1 && then->node->details == ((node_t *) then_rsc->running_on->data)->details) { /* ignore... if 'then' is supposed to be started after 'first', but * then is already started, there is nothing to be done when non-symmetrical. */ } else if (!(first->flags & pe_action_runnable)) { /* prevent 'then' action from happening if 'first' is not runnable and * 'then' has not yet occurred. */ pe_action_implies(then, first, pe_action_optional); pe_action_implies(then, first, pe_action_runnable); pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid); } else { /* ignore... then is allowed to start/stop if it wants to. */ } } if (type & pe_order_implies_first) { if (is_set(filter, pe_action_optional) && is_not_set(flags /* Should be then_flags? */, pe_action_optional)) { // Needs is_set(first_flags, pe_action_optional) too? pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid); pe_action_implies(first, then, pe_action_optional); } if (is_set(flags, pe_action_migrate_runnable) && is_set(then->flags, pe_action_migrate_runnable) == FALSE && is_set(then->flags, pe_action_optional) == FALSE) { pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid); pe_action_implies(first, then, pe_action_migrate_runnable); } } if (type & pe_order_implies_first_master) { if ((filter & pe_action_optional) && ((then->flags & pe_action_optional) == FALSE) && then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) { pe_action_implies(first, then, pe_action_optional); if (is_set(first->flags, pe_action_migrate_runnable) && is_set(then->flags, pe_action_migrate_runnable) == FALSE) { pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid); pe_action_implies(first, then, pe_action_migrate_runnable); } pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid); } } if ((type & pe_order_implies_first_migratable) && is_set(filter, pe_action_optional)) { if (((then->flags & pe_action_migrate_runnable) == FALSE) || ((then->flags & pe_action_runnable) == FALSE)) { pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid); pe_action_implies(first, then, pe_action_runnable); } if ((then->flags & pe_action_optional) == 0) { pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid); pe_action_implies(first, then, pe_action_optional); } } if ((type & pe_order_pseudo_left) && is_set(filter, pe_action_optional)) { if ((first->flags & pe_action_runnable) == FALSE) { pe_action_implies(then, first, pe_action_migrate_runnable); pe_clear_action_bit(then, pe_action_pseudo); pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid); } } if (is_set(type, pe_order_runnable_left) && is_set(filter, pe_action_runnable) && is_set(then->flags, pe_action_runnable) && is_set(flags, pe_action_runnable) == FALSE) { pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid); pe_action_implies(then, first, pe_action_runnable); pe_action_implies(then, first, pe_action_migrate_runnable); } if (is_set(type, pe_order_implies_then) && is_set(filter, pe_action_optional) && is_set(then->flags, pe_action_optional) && is_set(flags, pe_action_optional) == FALSE) { /* in this case, treat migrate_runnable as if first is optional */ if (is_set(first->flags, pe_action_migrate_runnable) == FALSE) { pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid); pe_action_implies(then, first, pe_action_optional); } } if (is_set(type, pe_order_restart)) { const char *reason = NULL; CRM_ASSERT(first->rsc && first->rsc->variant == pe_native); CRM_ASSERT(then->rsc && then->rsc->variant == pe_native); if ((filter & pe_action_runnable) && (then->flags & pe_action_runnable) == 0 && (then->rsc->flags & pe_rsc_managed)) { reason = "shutdown"; } if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) { reason = "recover"; } if (reason && is_set(first->flags, pe_action_optional)) { if (is_set(first->flags, pe_action_runnable) || is_not_set(then->flags, pe_action_optional)) { pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); pe_action_implies(first, then, pe_action_optional); } } if (reason && is_not_set(first->flags, pe_action_optional) && is_not_set(first->flags, pe_action_runnable)) { pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); pe_action_implies(then, first, pe_action_runnable); } if (reason && is_not_set(first->flags, pe_action_optional) && is_set(first->flags, pe_action_migrate_runnable) && is_not_set(then->flags, pe_action_migrate_runnable)) { pe_action_implies(first, then, pe_action_migrate_runnable); } } if (then_flags != then->flags) { changed |= pe_graph_updated_then; pe_rsc_trace(then->rsc, "Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x", then->uuid, then->node ? then->node->details->uname : "[none]", then->flags, then_flags, first->uuid, first->flags); if(then->rsc && then->rsc->parent) { /* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */ update_action(then); } } if (first_flags != first->flags) { changed |= pe_graph_updated_first; pe_rsc_trace(first->rsc, "First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x", first->uuid, first->node ? first->node->details->uname : "[none]", first->flags, first_flags, then->uuid, then->flags); } return changed; } void native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) { GListPtr gIter = NULL; GHashTableIter iter; node_t *node = NULL; if (constraint == NULL) { pe_err("Constraint is NULL"); return; } else if (rsc == NULL) { pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id); return; } pe_rsc_trace(rsc, "Applying %s (%s) to %s", constraint->id, role2text(constraint->role_filter), rsc->id); /* take "lifetime" into account */ if (constraint->role_filter > RSC_ROLE_UNKNOWN && constraint->role_filter != rsc->next_role) { pe_rsc_debug(rsc, "Constraint (%s) is not active (role : %s vs. %s)", constraint->id, role2text(constraint->role_filter), role2text(rsc->next_role)); return; } else if (is_active(constraint) == FALSE) { pe_rsc_trace(rsc, "Constraint (%s) is not active", constraint->id); return; } if (constraint->node_list_rh == NULL) { pe_rsc_trace(rsc, "RHS of constraint %s is NULL", constraint->id); return; } for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; node_t *other_node = NULL; other_node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id); if (other_node != NULL) { pe_rsc_trace(rsc, "%s + %s: %d + %d", node->details->uname, other_node->details->uname, node->weight, other_node->weight); other_node->weight = merge_weights(other_node->weight, node->weight); } else { other_node = node_copy(node); pe_rsc_trace(rsc, "%s: %d (insert %d)", other_node->details->uname, other_node->weight, constraint->discover_mode); g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node); } if (other_node->rsc_discover_mode < constraint->discover_mode) { if (constraint->discover_mode == discover_exclusive) { rsc->exclusive_discover = TRUE; } /* exclusive > never > always... always is default */ other_node->rsc_discover_mode = constraint->discover_mode; } } g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { pe_rsc_trace(rsc, "%s + %s : %d", rsc->id, node->details->uname, node->weight); } } void native_expand(resource_t * rsc, pe_working_set_t * data_set) { GListPtr gIter = NULL; CRM_ASSERT(rsc); pe_rsc_trace(rsc, "Processing actions from %s", rsc->id); for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; crm_trace("processing action %d for rsc=%s", action->id, rsc->id); graph_element_from_action(action, data_set); } for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; child_rsc->cmds->expand(child_rsc, data_set); } } #define log_change(a, fmt, args...) do { \ if(a && a->reason && terminal) { \ printf(" * "fmt" \tdue to %s\n", ##args, a->reason); \ } else if(a && a->reason) { \ crm_notice(fmt" \tdue to %s", ##args, a->reason); \ } else if(terminal) { \ printf(" * "fmt"\n", ##args); \ } else { \ crm_notice(fmt, ##args); \ } \ } while(0) #define STOP_SANITY_ASSERT(lineno) do { \ if(current && current->details->unclean) { \ /* It will be a pseudo op */ \ } else if(stop == NULL) { \ crm_err("%s:%d: No stop action exists for %s", __FUNCTION__, lineno, rsc->id); \ CRM_ASSERT(stop != NULL); \ } else if(is_set(stop->flags, pe_action_optional)) { \ crm_err("%s:%d: Action %s is still optional", __FUNCTION__, lineno, stop->uuid); \ CRM_ASSERT(is_not_set(stop->flags, pe_action_optional)); \ } \ } while(0) static int rsc_width = 5; static int detail_width = 5; static void LogAction(const char *change, resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal) { int len = 0; char *reason = NULL; char *details = NULL; bool same_host = FALSE; bool same_role = FALSE; bool need_role = FALSE; CRM_ASSERT(action); CRM_ASSERT(destination != NULL || origin != NULL); if(source == NULL) { source = action; } len = strlen(rsc->id); if(len > rsc_width) { rsc_width = len + 2; } if(rsc->role > RSC_ROLE_STARTED || rsc->next_role > RSC_ROLE_SLAVE) { need_role = TRUE; } if(origin != NULL && destination != NULL && origin->details == destination->details) { same_host = TRUE; } if(rsc->role == rsc->next_role) { same_role = TRUE; } if(need_role && origin == NULL) { /* Promoting from Stopped */ details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname); } else if(need_role && destination == NULL) { /* Demoting a Master or Stopping a Slave */ details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); } else if(origin == NULL || destination == NULL) { /* Starting or stopping a resource */ details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname); } else if(need_role && same_role && same_host) { /* Recovering or Restarting a Master/Slave resource */ details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); } else if(same_role && same_host) { /* Recovering or Restarting a normal resource */ details = crm_strdup_printf("%s", origin->details->uname); } else if(same_role && need_role) { /* Moving a Master/Slave resource */ details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role)); } else if(same_role) { /* Moving a normal resource */ details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname); } else if(same_host) { /* Promoting or Demoting a Master/Slave resource */ details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname); } else { /* Moving and promoting/demoting */ details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role), origin->details->uname, role2text(rsc->next_role), destination->details->uname); } len = strlen(details); if(len > detail_width) { detail_width = len; } if(source->reason && is_not_set(action->flags, pe_action_runnable)) { reason = crm_strdup_printf(" due to %s (blocked)", source->reason); } else if(source->reason) { reason = crm_strdup_printf(" due to %s", source->reason); } else if(is_not_set(action->flags, pe_action_runnable)) { reason = strdup(" blocked"); } else { reason = strdup(""); } if(terminal) { printf(" * %-8s %-*s ( %*s ) %s\n", change, rsc_width, rsc->id, detail_width, details, reason); } else { crm_notice(" * %-8s %-*s ( %*s ) %s", change, rsc_width, rsc->id, detail_width, details, reason); } free(details); free(reason); } void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) { node_t *next = NULL; node_t *current = NULL; action_t *stop = NULL; action_t *start = NULL; action_t *demote = NULL; action_t *promote = NULL; char *key = NULL; gboolean moving = FALSE; GListPtr possible_matches = NULL; if(rsc->variant == pe_container) { container_LogActions(rsc, data_set, terminal); return; } if (rsc->children) { GListPtr gIter = NULL; for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; LogActions(child_rsc, data_set, terminal); } return; } next = rsc->allocated_to; if (rsc->running_on) { if (g_list_length(rsc->running_on) > 1 && rsc->partial_migration_source) { current = rsc->partial_migration_source; } else { current = rsc->running_on->data; } if (rsc->role == RSC_ROLE_STOPPED) { /* * This can occur when resources are being recovered * We fiddle with the current role in native_create_actions() */ rsc->role = RSC_ROLE_STARTED; } } if (current == NULL && is_set(rsc->flags, pe_rsc_orphan)) { /* Don't log stopped orphans */ return; } if (is_not_set(rsc->flags, pe_rsc_managed) || (current == NULL && next == NULL)) { pe_rsc_info(rsc, "Leave %s\t(%s%s)", rsc->id, role2text(rsc->role), is_not_set(rsc->flags, pe_rsc_managed) ? " unmanaged" : ""); return; } if (current != NULL && next != NULL && safe_str_neq(current->details->id, next->details->id)) { moving = TRUE; } key = start_key(rsc); possible_matches = find_actions(rsc->actions, key, next); free(key); if (possible_matches) { start = possible_matches->data; g_list_free(possible_matches); } key = stop_key(rsc); if(start == NULL || is_set(start->flags, pe_action_runnable) == FALSE) { possible_matches = find_actions(rsc->actions, key, NULL); } else { possible_matches = find_actions(rsc->actions, key, current); } if (possible_matches) { stop = possible_matches->data; g_list_free(possible_matches); } free(key); key = promote_key(rsc); possible_matches = find_actions(rsc->actions, key, next); free(key); if (possible_matches) { promote = possible_matches->data; g_list_free(possible_matches); } key = demote_key(rsc); possible_matches = find_actions(rsc->actions, key, next); free(key); if (possible_matches) { demote = possible_matches->data; g_list_free(possible_matches); } if (rsc->role == rsc->next_role) { action_t *migrate_to = NULL; key = generate_op_key(rsc->id, RSC_MIGRATED, 0); possible_matches = find_actions(rsc->actions, key, next); free(key); if (possible_matches) { migrate_to = possible_matches->data; } CRM_CHECK(next != NULL,); if (next == NULL) { } else if (migrate_to && is_set(migrate_to->flags, pe_action_runnable) && current) { LogAction("Migrate", rsc, current, next, start, NULL, terminal); } else if (is_set(rsc->flags, pe_rsc_reload)) { LogAction("Reload", rsc, current, next, start, NULL, terminal); } else if (start == NULL || is_set(start->flags, pe_action_optional)) { pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) { LogAction("Stop", rsc, current, NULL, stop, start, terminal); STOP_SANITY_ASSERT(__LINE__); } else if (moving && current) { LogAction(is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move", rsc, current, next, stop, NULL, terminal); } else if (is_set(rsc->flags, pe_rsc_failed)) { LogAction("Recover", rsc, current, NULL, stop, NULL, terminal); STOP_SANITY_ASSERT(__LINE__); } else { LogAction("Restart", rsc, current, next, start, NULL, terminal); /* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */ } g_list_free(possible_matches); return; } if(stop && (rsc->next_role == RSC_ROLE_STOPPED || (start && is_not_set(start->flags, pe_action_runnable)))) { GListPtr gIter = NULL; key = stop_key(rsc); for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; action_t *stop_op = NULL; possible_matches = find_actions(rsc->actions, key, node); if (possible_matches) { stop_op = possible_matches->data; g_list_free(possible_matches); } if (stop_op && (stop_op->flags & pe_action_runnable)) { STOP_SANITY_ASSERT(__LINE__); } LogAction("Stop", rsc, node, NULL, stop_op, start, terminal); } free(key); } else if (stop && is_set(rsc->flags, pe_rsc_failed)) { /* 'stop' may be NULL if the failure was ignored */ LogAction("Recover", rsc, current, next, stop, start, terminal); STOP_SANITY_ASSERT(__LINE__); } else if (moving) { LogAction("Move", rsc, current, next, stop, NULL, terminal); STOP_SANITY_ASSERT(__LINE__); } else if (is_set(rsc->flags, pe_rsc_reload)) { LogAction("Reload", rsc, current, next, start, NULL, terminal); } else if (stop != NULL && is_not_set(stop->flags, pe_action_optional)) { LogAction("Restart", rsc, current, next, start, NULL, terminal); STOP_SANITY_ASSERT(__LINE__); } else if (rsc->role == RSC_ROLE_MASTER) { CRM_LOG_ASSERT(current != NULL); LogAction("Demote", rsc, current, next, demote, NULL, terminal); } else if(rsc->next_role == RSC_ROLE_MASTER) { CRM_LOG_ASSERT(next); LogAction("Promote", rsc, current, next, promote, NULL, terminal); } else if (rsc->role == RSC_ROLE_STOPPED && rsc->next_role > RSC_ROLE_STOPPED) { LogAction("Start", rsc, current, next, start, NULL, terminal); } } gboolean StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { GListPtr gIter = NULL; CRM_ASSERT(rsc); pe_rsc_trace(rsc, "%s", rsc->id); for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { node_t *current = (node_t *) gIter->data; action_t *stop; if (rsc->partial_migration_target) { if (rsc->partial_migration_target->details == current->details) { pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname, next->details->uname, rsc->id); continue; } else { pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id); optional = FALSE; } } pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname); stop = stop_action(rsc, current, optional); if (is_not_set(rsc->flags, pe_rsc_managed)) { update_action_flags(stop, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } if (is_set(data_set->flags, pe_flag_remove_after_stop)) { DeleteRsc(rsc, current, optional, data_set); } if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); const char *unfenced = node_attribute_raw(current, XML_NODE_IS_UNFENCED); order_actions(stop, unfence, pe_order_implies_first); if (unfenced == NULL || safe_str_eq("0", unfenced)) { pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname); } } } return TRUE; } gboolean StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { action_t *start = NULL; CRM_ASSERT(rsc); pe_rsc_trace(rsc, "%s on %s %d", rsc->id, next ? next->details->uname : "N/A", optional); start = start_action(rsc, next, TRUE); if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set); const char *unfenced = node_attribute_raw(next, XML_NODE_IS_UNFENCED); order_actions(unfence, start, pe_order_implies_then); if (unfenced == NULL || safe_str_eq("0", unfenced)) { char *reason = crm_strdup_printf("Required by %s", rsc->id); trigger_unfencing(NULL, next, reason, NULL, data_set); free(reason); } } if (is_set(start->flags, pe_action_runnable) && optional == FALSE) { update_action_flags(start, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); } return TRUE; } gboolean PromoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { char *key = NULL; GListPtr gIter = NULL; gboolean runnable = TRUE; GListPtr action_list = NULL; CRM_ASSERT(rsc); CRM_CHECK(next != NULL, return FALSE); pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname); key = start_key(rsc); action_list = find_actions_exact(rsc->actions, key, next); free(key); for (gIter = action_list; gIter != NULL; gIter = gIter->next) { action_t *start = (action_t *) gIter->data; if (is_set(start->flags, pe_action_runnable) == FALSE) { runnable = FALSE; } } g_list_free(action_list); if (runnable) { promote_action(rsc, next, optional); return TRUE; } pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id); key = promote_key(rsc); action_list = find_actions_exact(rsc->actions, key, next); free(key); for (gIter = action_list; gIter != NULL; gIter = gIter->next) { action_t *promote = (action_t *) gIter->data; update_action_flags(promote, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__); } g_list_free(action_list); return TRUE; } gboolean DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { GListPtr gIter = NULL; CRM_ASSERT(rsc); pe_rsc_trace(rsc, "%s", rsc->id); /* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */ for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { node_t *current = (node_t *) gIter->data; pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A"); demote_action(rsc, current, optional); } return TRUE; } gboolean RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { CRM_ASSERT(rsc); crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A"); CRM_CHECK(FALSE, return FALSE); return FALSE; } gboolean NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set) { CRM_ASSERT(rsc); pe_rsc_trace(rsc, "%s", rsc->id); return FALSE; } gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set) { if (is_set(rsc->flags, pe_rsc_failed)) { pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname); return FALSE; } else if (node == NULL) { pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id); return FALSE; } else if (node->details->unclean || node->details->online == FALSE) { pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id, node->details->uname); return FALSE; } crm_notice("Removing %s from %s", rsc->id, node->details->uname); delete_action(rsc, node, optional); new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE, optional ? pe_order_implies_then : pe_order_optional, data_set); new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START, optional ? pe_order_implies_then : pe_order_optional, data_set); return TRUE; } #include <../lib/pengine/unpack.h> #define set_char(x) last_rsc_id[lpc] = x; complete = TRUE; static char * increment_clone(char *last_rsc_id) { int lpc = 0; int len = 0; char *tmp = NULL; gboolean complete = FALSE; CRM_CHECK(last_rsc_id != NULL, return NULL); if (last_rsc_id != NULL) { len = strlen(last_rsc_id); } lpc = len - 1; while (complete == FALSE && lpc > 0) { switch (last_rsc_id[lpc]) { case 0: lpc--; break; case '0': set_char('1'); break; case '1': set_char('2'); break; case '2': set_char('3'); break; case '3': set_char('4'); break; case '4': set_char('5'); break; case '5': set_char('6'); break; case '6': set_char('7'); break; case '7': set_char('8'); break; case '8': set_char('9'); break; case '9': last_rsc_id[lpc] = '0'; lpc--; break; case ':': tmp = last_rsc_id; last_rsc_id = calloc(1, len + 2); memcpy(last_rsc_id, tmp, len); last_rsc_id[++lpc] = '1'; last_rsc_id[len] = '0'; last_rsc_id[len + 1] = 0; complete = TRUE; free(tmp); break; default: crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc); return NULL; break; } } return last_rsc_id; } static node_t * probe_grouped_clone(resource_t * rsc, node_t * node, pe_working_set_t * data_set) { node_t *running = NULL; resource_t *top = uber_parent(rsc); if (running == NULL && is_set(top->flags, pe_rsc_unique) == FALSE) { /* Annoyingly we also need to check any other clone instances * Clumsy, but it will work. * * An alternative would be to update known_on for every peer * during process_rsc_state() * * This code desperately needs optimization * ptest -x with 100 nodes, 100 clones and clone-max=10: * No probes O(25s) * Detection without clone loop O(3m) * Detection with clone loop O(8m) ptest[32211]: 2010/02/18_14:27:55 CRIT: stage5: Probing for unknown resources ptest[32211]: 2010/02/18_14:33:39 CRIT: stage5: Done ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Updating action states ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Done */ char *clone_id = clone_zero(rsc->id); resource_t *peer = pe_find_resource(top->children, clone_id); while (peer && running == NULL) { running = pe_hash_table_lookup(peer->known_on, node->details->id); if (running != NULL) { /* we already know the status of the resource on this node */ pe_rsc_trace(rsc, "Skipping active clone: %s", rsc->id); free(clone_id); return running; } clone_id = increment_clone(clone_id); peer = pe_find_resource(data_set->resources, clone_id); } free(clone_id); } return running; } gboolean native_create_probe(resource_t * rsc, node_t * node, action_t * complete, gboolean force, pe_working_set_t * data_set) { enum pe_ordering flags = pe_order_optional; char *key = NULL; action_t *probe = NULL; node_t *running = NULL; node_t *allowed = NULL; resource_t *top = uber_parent(rsc); static const char *rc_master = NULL; static const char *rc_inactive = NULL; if (rc_inactive == NULL) { rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING); rc_master = crm_itoa(PCMK_OCF_RUNNING_MASTER); } CRM_CHECK(node != NULL, return FALSE); if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) { pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id); return FALSE; - } else if (force == FALSE && is_container_remote_node(node)) { - pe_rsc_trace(rsc, "Skipping active resource detection for %s on container %s", - rsc->id, node->details->id); - return FALSE; } if (is_remote_node(node)) { const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) { pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes do not run stonith agents.", rsc->id, node->details->id); return FALSE; } else if (rsc_contains_remote_node(data_set, rsc)) { pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes can not run resources that contain connection resources.", rsc->id, node->details->id); return FALSE; } else if (rsc->is_remote_node) { pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes can not run connection resources", rsc->id, node->details->id); return FALSE; } } if (rsc->children) { GListPtr gIter = NULL; gboolean any_created = FALSE; for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set) || any_created; } return any_created; } else if ((rsc->container) && (!rsc->is_remote_node)) { pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id); return FALSE; } if (is_set(rsc->flags, pe_rsc_orphan)) { pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id); return FALSE; } running = g_hash_table_lookup(rsc->known_on, node->details->id); if (running == NULL && is_set(rsc->flags, pe_rsc_unique) == FALSE) { /* Anonymous clones */ if (rsc->parent == top) { running = g_hash_table_lookup(rsc->parent->known_on, node->details->id); } else { /* Grouped anonymous clones need extra special handling */ running = probe_grouped_clone(rsc, node, data_set); } } if (force == FALSE && running != NULL) { /* we already know the status of the resource on this node */ pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname); return FALSE; } allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); + if (rsc->exclusive_discover || top->exclusive_discover) { if (allowed == NULL) { /* exclusive discover is enabled and this node is not in the allowed list. */ return FALSE; } else if (allowed->rsc_discover_mode != discover_exclusive) { /* exclusive discover is enabled and this node is not marked * as a node this resource should be discovered on */ return FALSE; } } + + if(allowed == NULL && node->rsc_discover_mode == discover_never) { + /* If this node was allowed to host this resource it would + * have been explicitly added to the 'allowed_nodes' list. + * However it wasn't and the node has discovery disabled, so + * no need to probe for this resource. + */ + return FALSE; + } + if (allowed && allowed->rsc_discover_mode == discover_never) { /* this resource is marked as not needing to be discovered on this node */ return FALSE; } + if(allowed != NULL && is_container_remote_node(allowed)) { + resource_t *remote = allowed->details->remote_rsc; + + if(remote->role == RSC_ROLE_STOPPED) { + /* If the container is stopped, then we know anything that + * might have been inside it is also stopped and there is + * no need to probe. + * + * If we don't know the container's state on the target + * either: + * + * - the container is running, the transition will abort + * and we'll end up in a different case next time, or + * + * - the container is stopped + * + * Either way there is no need to probe. + * + */ + if(remote->allocated_to + && g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) { + /* For safety, we order the 'rsc' start after 'remote' + * has been probed. + * + * Using 'top' helps for groups, but in we may need to + * follow the start's ordering chain backwards. + */ + custom_action_order(remote, generate_op_key(remote->id, RSC_STATUS, 0), NULL, + top, generate_op_key(top->id, RSC_START, 0), NULL, + pe_order_optional, data_set); + } + return FALSE; + + /* Here we really we want to check if remote->stop is required, + * but that information doesn't exist yet + */ + } else if(allowed->details->remote_requires_reset + || allowed->details->unclean + || is_set(remote->flags, pe_rsc_failed) + || remote->next_role == RSC_ROLE_STOPPED + || (remote->allocated_to + && pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL) + ) { + /* The container is stopping or restarting, don't start + * 'rsc' until 'remote' stops as this also implies that + * 'rsc' is stopped - avoiding the need to probe + */ + custom_action_order(remote, generate_op_key(remote->id, RSC_STOP, 0), NULL, + top, generate_op_key(top->id, RSC_START, 0), NULL, + pe_order_optional, data_set); + return FALSE; +/* } else { + * The container is running so there is no problem probing it + */ + } + } key = generate_op_key(rsc->id, RSC_STATUS, 0); probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set); update_action_flags(probe, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); /* If enabled, require unfencing before probing any fence devices * but ensure it happens after any resources that require * unfencing have been probed. * * Doing it the other way (requiring unfencing after probing * resources that need it) would result in the node being * unfenced, and all its resources being stopped, whenever a new * resource is added. Which would be highly suboptimal. * * So essentially, at the point the fencing device(s) have been * probed, we know the state of all resources that require * unfencing and that unfencing occurred. */ if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); order_actions(unfence, probe, pe_order_optional); } /* * We need to know if it's running_on (not just known_on) this node * to correctly determine the target rc. */ running = pe_find_node_id(rsc->running_on, node->details->id); if (running == NULL) { add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive); } else if (rsc->role == RSC_ROLE_MASTER) { add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master); } crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role), is_set(probe->flags, pe_action_runnable), rsc->running_on); if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { top = rsc; } else if (pe_rsc_is_clone(top) == FALSE) { top = rsc; } else { crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id); } if(is_not_set(probe->flags, pe_action_runnable) && rsc->running_on == NULL) { /* Prevent the start from occurring if rsc isn't active, but * don't cause it to stop if it was active already */ flags |= pe_order_runnable_left; } custom_action_order(rsc, NULL, probe, top, generate_op_key(top->id, RSC_START, 0), NULL, flags, data_set); /* Before any reloads, if they exist */ custom_action_order(rsc, NULL, probe, top, reload_key(rsc), NULL, pe_order_optional, data_set); if (node->details->shutdown == FALSE) { custom_action_order(rsc, NULL, probe, rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_optional, data_set); } if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { /* Normally rsc.start depends on probe complete which depends * on rsc.probe. But this can't be the case in this scenario as * it would create graph loops. * * So instead we explicitly order 'rsc.probe then rsc.start' */ } else { order_actions(probe, complete, pe_order_implies_then); } return TRUE; } static void native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set) { node_t *target; GListPtr gIter = NULL; action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set); action_t *stonith_done = get_pseudo_op(STONITH_DONE, data_set); CRM_CHECK(stonith_op && stonith_op->node, return); target = stonith_op->node; for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if(action->needs == rsc_req_nothing) { /* Anything other than start or promote requires nothing */ } else if (action->needs == rsc_req_stonith) { order_actions(stonith_done, action, pe_order_optional); } else if (safe_str_eq(action->task, RSC_START) && NULL == pe_hash_table_lookup(rsc->known_on, target->details->id)) { /* if known == NULL, then we don't know if * the resource is active on the node * we're about to shoot * * in this case, regardless of action->needs, * the only safe option is to wait until * the node is shot before doing anything * to with the resource * * it's analogous to waiting for all the probes * for rscX to complete before starting rscX * * the most likely explanation is that the * DC died and took its status with it */ pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid, target->details->uname); order_actions(all_stopped, action, pe_order_optional | pe_order_runnable_left); } } } static void native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set) { char *key = NULL; GListPtr gIter = NULL; GListPtr action_list = NULL; action_t *start = NULL; resource_t *top = uber_parent(rsc); node_t *target; CRM_CHECK(stonith_op && stonith_op->node, return); target = stonith_op->node; /* Check whether the resource has a pending start action */ start = find_first_action(rsc->actions, NULL, CRMD_ACTION_START, NULL); /* Get a list of stop actions potentially implied by the fencing */ key = stop_key(rsc); action_list = find_actions(rsc->actions, key, target); free(key); for (gIter = action_list; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (is_set(rsc->flags, pe_rsc_failed)) { crm_notice("Stop of failed resource %s is implicit after %s is fenced", rsc->id, target->details->uname); } else { crm_info("%s is implicit after %s is fenced", action->uuid, target->details->uname); } /* The stop would never complete and is now implied by the fencing, * so convert it into a pseudo-action. */ update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__); update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); update_action_flags(action, pe_action_implied_by_stonith, __FUNCTION__, __LINE__); if(start == NULL || start->needs > rsc_req_quorum) { enum pe_ordering flags = pe_order_optional; action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL); if (target->details->remote_rsc) { /* User constraints must not order a resource in a guest node * relative to the guest node container resource. This flag * marks constraints as generated by the cluster and thus * immune to that check. */ flags |= pe_order_preserve; } order_actions(stonith_op, action, flags); order_actions(stonith_op, parent_stop, flags); } if (is_set(rsc->flags, pe_rsc_notify)) { /* Create a second notification that will be delivered * immediately after the node is fenced * * Basic problem: * - C is a clone active on the node to be shot and stopping on another * - R is a resource that depends on C * * + C.stop depends on R.stop * + C.stopped depends on STONITH * + C.notify depends on C.stopped * + C.healthy depends on C.notify * + R.stop depends on C.healthy * * The extra notification here changes * + C.healthy depends on C.notify * into: * + C.healthy depends on C.notify' * + C.notify' depends on STONITH' * thus breaking the loop */ create_secondary_notification(action, rsc, stonith_op, data_set); } /* From Bug #1601, successful fencing must be an input to a failed resources stop action. However given group(rA, rB) running on nodeX and B.stop has failed, A := stop healthy resource (rA.stop) B := stop failed resource (pseudo operation B.stop) C := stonith nodeX A requires B, B requires C, C requires A This loop would prevent the cluster from making progress. This block creates the "C requires A" dependency and therefore must (at least for now) be disabled. Instead, run the block above and treat all resources on nodeX as B would be (marked as a pseudo op depending on the STONITH). TODO: Break the "A requires B" dependency in update_action() and re-enable this block } else if(is_stonith == FALSE) { crm_info("Moving healthy resource %s" " off %s before fencing", rsc->id, node->details->uname); * stop healthy resources before the * stonith op * custom_action_order( rsc, stop_key(rsc), NULL, NULL,strdup(CRM_OP_FENCE),stonith_op, pe_order_optional, data_set); */ } g_list_free(action_list); /* Get a list of demote actions potentially implied by the fencing */ key = demote_key(rsc); action_list = find_actions(rsc->actions, key, target); free(key); for (gIter = action_list; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (action->node->details->online == FALSE || action->node->details->unclean == TRUE || is_set(rsc->flags, pe_rsc_failed)) { if (is_set(rsc->flags, pe_rsc_failed)) { pe_rsc_info(rsc, "Demote of failed resource %s is implicit after %s is fenced", rsc->id, target->details->uname); } else { pe_rsc_info(rsc, "%s is implicit after %s is fenced", action->uuid, target->details->uname); } /* The demote would never complete and is now implied by the * fencing, so convert it into a pseudo-action. */ update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__); update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); if (start == NULL || start->needs > rsc_req_quorum) { order_actions(stonith_op, action, pe_order_preserve|pe_order_optional); } } } g_list_free(action_list); } void rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set) { if (rsc->children) { GListPtr gIter = NULL; for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; rsc_stonith_ordering(child_rsc, stonith_op, data_set); } } else if (is_not_set(rsc->flags, pe_rsc_managed)) { pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id); } else { native_start_constraints(rsc, stonith_op, data_set); native_stop_constraints(rsc, stonith_op, data_set); } } enum stack_activity { stack_stable = 0, stack_starting = 1, stack_stopping = 2, stack_middle = 4, }; static action_t * get_first_named_action(resource_t * rsc, const char *action, gboolean only_valid, node_t * current) { action_t *a = NULL; GListPtr action_list = NULL; char *key = generate_op_key(rsc->id, action, 0); action_list = find_actions(rsc->actions, key, current); if (action_list == NULL || action_list->data == NULL) { crm_trace("%s: no %s action", rsc->id, action); free(key); return NULL; } a = action_list->data; g_list_free(action_list); if (only_valid && is_set(a->flags, pe_action_pseudo)) { crm_trace("%s: pseudo", key); a = NULL; } else if (only_valid && is_not_set(a->flags, pe_action_runnable)) { crm_trace("%s: runnable", key); a = NULL; } free(key); return a; } void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set) { GListPtr gIter = NULL; action_t *other = NULL; action_t *reload = NULL; if (rsc->children) { for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; ReloadRsc(child_rsc, node, data_set); } return; } else if (rsc->variant > pe_native) { /* Complex resource with no children */ return; } else if (is_not_set(rsc->flags, pe_rsc_managed)) { pe_rsc_trace(rsc, "%s: unmanaged", rsc->id); return; } else if (is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending)) { pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags); stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */ return; } else if (node == NULL) { pe_rsc_trace(rsc, "%s: not active", rsc->id); return; } pe_rsc_trace(rsc, "Processing %s", rsc->id); set_bit(rsc->flags, pe_rsc_reload); reload = custom_action( rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set); /* stop = stop_action(rsc, node, optional); */ other = get_first_named_action(rsc, RSC_STOP, TRUE, node); if (other != NULL) { order_actions(reload, other, pe_order_optional); } other = get_first_named_action(rsc, RSC_DEMOTE, TRUE, node); if (other != NULL) { order_actions(reload, other, pe_order_optional); } } void native_append_meta(resource_t * rsc, xmlNode * xml) { char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION); resource_t *iso_parent, *last_parent, *parent; if (value) { char *name = NULL; name = crm_meta_name(XML_RSC_ATTR_INCARNATION); crm_xml_add(xml, name, value); free(name); } value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE); if (value) { char *name = NULL; name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE); crm_xml_add(xml, name, value); free(name); } for (parent = rsc; parent != NULL; parent = parent->parent) { if (parent->container) { crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id); } } last_parent = iso_parent = rsc; while (iso_parent != NULL) { char *name = NULL; char *iso = NULL; if (iso_parent->isolation_wrapper == NULL) { last_parent = iso_parent; iso_parent = iso_parent->parent; continue; } /* name of wrapper script this resource is routed through. */ name = crm_meta_name(XML_RSC_ATTR_ISOLATION_WRAPPER); crm_xml_add(xml, name, iso_parent->isolation_wrapper); free(name); /* instance name for isolated environment */ name = crm_meta_name(XML_RSC_ATTR_ISOLATION_INSTANCE); if (pe_rsc_is_clone(iso_parent)) { /* if isolation is set at the clone/master level, we have to * give this resource the unique isolation instance associated * with the clone child (last_parent)*/ /* Example: cloned group. group is container * clone myclone - iso_parent * group mygroup - last_parent (this is the iso environment) * rsc myrsc1 - rsc * rsc myrsc2 * The group is what is isolated in example1. We have to make * sure myrsc1 and myrsc2 launch in the same isolated environment. * * Example: cloned primitives. rsc primitive is container * clone myclone iso_parent * rsc myrsc1 - last_parent == rsc (this is the iso environment) * The individual cloned primitive instances are isolated */ value = g_hash_table_lookup(last_parent->meta, XML_RSC_ATTR_INCARNATION); CRM_ASSERT(value != NULL); iso = crm_concat(crm_element_value(last_parent->xml, XML_ATTR_ID), value, '_'); crm_xml_add(xml, name, iso); free(iso); } else { /* * Example: cloned group of containers * clone myclone * group mygroup * rsc myrsc1 - iso_parent (this is the iso environment) * rsc myrsc2 * * Example: group of containers * group mygroup * rsc myrsc1 - iso_parent (this is the iso environment) * rsc myrsc2 * * Example: group is container * group mygroup - iso_parent ( this is iso environment) * rsc myrsc1 * rsc myrsc2 * * Example: single primitive * rsc myrsc1 - iso_parent (this is the iso environment) */ value = g_hash_table_lookup(iso_parent->meta, XML_RSC_ATTR_INCARNATION); if (value) { crm_xml_add(xml, name, iso_parent->id); iso = crm_concat(crm_element_value(iso_parent->xml, XML_ATTR_ID), value, '_'); crm_xml_add(xml, name, iso); free(iso); } else { crm_xml_add(xml, name, iso_parent->id); } } free(name); break; } } diff --git a/pengine/pengine.h b/pengine/pengine.h index e3f4874eb7..2c13258ae5 100644 --- a/pengine/pengine.h +++ b/pengine/pengine.h @@ -1,156 +1,150 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef PENGINE__H # define PENGINE__H typedef struct rsc_to_node_s rsc_to_node_t; typedef struct rsc_colocation_s rsc_colocation_t; typedef struct rsc_ticket_s rsc_ticket_t; typedef struct lrm_agent_s lrm_agent_t; typedef struct order_constraint_s order_constraint_t; # include # include # include # include # include # include # include enum pe_stop_fail { pesf_block, pesf_stonith, pesf_ignore }; enum pe_weights { pe_weights_none = 0x0, pe_weights_init = 0x1, pe_weights_forward = 0x4, pe_weights_positive = 0x8, pe_weights_rollback = 0x10, }; struct rsc_colocation_s { const char *id; const char *node_attribute; resource_t *rsc_lh; resource_t *rsc_rh; int role_lh; int role_rh; int score; }; enum loss_ticket_policy_e { loss_ticket_stop, loss_ticket_demote, loss_ticket_fence, loss_ticket_freeze }; struct rsc_ticket_s { const char *id; resource_t *rsc_lh; ticket_t *ticket; enum loss_ticket_policy_e loss_policy; int role_lh; }; -enum rsc_discover_e { - discover_always = 0, - discover_never, - discover_exclusive, -}; - struct rsc_to_node_s { char *id; resource_t *rsc_lh; enum rsc_role_e role_filter; enum rsc_discover_e discover_mode; GListPtr node_list_rh; /* node_t* */ }; struct order_constraint_s { int id; enum pe_ordering type; void *lh_opaque; resource_t *lh_rsc; action_t *lh_action; char *lh_action_task; void *rh_opaque; resource_t *rh_rsc; action_t *rh_action; char *rh_action_task; /* (soon to be) variant specific */ /* int lh_rsc_incarnation; */ /* int rh_rsc_incarnation; */ }; extern gboolean stage0(pe_working_set_t * data_set); extern gboolean probe_resources(pe_working_set_t * data_set); extern gboolean stage2(pe_working_set_t * data_set); extern gboolean stage3(pe_working_set_t * data_set); extern gboolean stage4(pe_working_set_t * data_set); extern gboolean stage5(pe_working_set_t * data_set); extern gboolean stage6(pe_working_set_t * data_set); extern gboolean stage7(pe_working_set_t * data_set); extern gboolean stage8(pe_working_set_t * data_set); extern gboolean summary(GListPtr resources); extern gboolean unpack_constraints(xmlNode * xml_constraints, pe_working_set_t * data_set); extern gboolean update_action_states(GListPtr actions); extern gboolean shutdown_constraints(node_t * node, action_t * shutdown_op, pe_working_set_t * data_set); extern gboolean stonith_constraints(node_t * node, action_t * stonith_op, pe_working_set_t * data_set); extern int custom_action_order(resource_t * lh_rsc, char *lh_task, action_t * lh_action, resource_t * rh_rsc, char *rh_task, action_t * rh_action, enum pe_ordering type, pe_working_set_t * data_set); extern int new_rsc_order(resource_t * lh_rsc, const char *lh_task, resource_t * rh_rsc, const char *rh_task, enum pe_ordering type, pe_working_set_t * data_set); # define order_start_start(rsc1,rsc2, type) \ new_rsc_order(rsc1, CRMD_ACTION_START, rsc2, CRMD_ACTION_START, type, data_set) # define order_stop_stop(rsc1, rsc2, type) \ new_rsc_order(rsc1, CRMD_ACTION_STOP, rsc2, CRMD_ACTION_STOP, type, data_set) extern void graph_element_from_action(action_t * action, pe_working_set_t * data_set); extern void add_maintenance_update(pe_working_set_t *data_set); extern gboolean show_scores; extern int scores_log_level; extern gboolean show_utilization; extern int utilization_log_level; extern const char *transition_idle_timeout; #endif diff --git a/pengine/test10/bug-cl-5247.dot b/pengine/test10/bug-cl-5247.dot index 44f788dcef..decf5c3180 100644 --- a/pengine/test10/bug-cl-5247.dot +++ b/pengine/test10/bug-cl-5247.dot @@ -1,148 +1,154 @@ digraph "g" { "all_stopped" -> "prmStonith1-2_start_0 bl460g8n4" [ style = bold] "all_stopped" -> "prmStonith2-2_start_0 bl460g8n3" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "grpStonith1_running_0" [ style=bold color="green" fontcolor="orange"] "grpStonith1_start_0" -> "grpStonith1_running_0" [ style = bold] "grpStonith1_start_0" -> "prmStonith1-2_start_0 bl460g8n4" [ style = bold] "grpStonith1_start_0" [ style=bold color="green" fontcolor="orange"] "grpStonith1_stop_0" -> "grpStonith1_stopped_0" [ style = bold] "grpStonith1_stop_0" -> "prmStonith1-2_stop_0 bl460g8n4" [ style = bold] "grpStonith1_stop_0" [ style=bold color="green" fontcolor="orange"] "grpStonith1_stopped_0" -> "grpStonith1_start_0" [ style = bold] "grpStonith1_stopped_0" [ style=bold color="green" fontcolor="orange"] "grpStonith2_running_0" [ style=bold color="green" fontcolor="orange"] "grpStonith2_start_0" -> "grpStonith2_running_0" [ style = bold] "grpStonith2_start_0" -> "prmStonith2-2_start_0 bl460g8n3" [ style = bold] "grpStonith2_start_0" [ style=bold color="green" fontcolor="orange"] "grpStonith2_stop_0" -> "grpStonith2_stopped_0" [ style = bold] "grpStonith2_stop_0" -> "prmStonith2-2_stop_0 bl460g8n3" [ style = bold] "grpStonith2_stop_0" [ style=bold color="green" fontcolor="orange"] "grpStonith2_stopped_0" -> "grpStonith2_start_0" [ style = bold] "grpStonith2_stopped_0" [ style=bold color="green" fontcolor="orange"] "master-group_running_0" [ style=bold color="green" fontcolor="orange"] "master-group_start_0" -> "master-group_running_0" [ style = bold] "master-group_start_0" -> "vip-master_start_0 pgsr01" [ style = bold] "master-group_start_0" -> "vip-rep_start_0 pgsr01" [ style = bold] "master-group_start_0" [ style=bold color="green" fontcolor="orange"] "master-group_stop_0" -> "master-group_stopped_0" [ style = bold] "master-group_stop_0" -> "vip-master_stop_0 pgsr02" [ style = bold] "master-group_stop_0" -> "vip-rep_stop_0 pgsr02" [ style = bold] "master-group_stop_0" [ style=bold color="green" fontcolor="orange"] "master-group_stopped_0" -> "master-group_start_0" [ style = bold] "master-group_stopped_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_confirmed-post_notify_demoted_0" -> "master-group_stop_0" [ style = bold] "msPostgresql_confirmed-post_notify_demoted_0" -> "msPostgresql_pre_notify_stop_0" [ style = bold] "msPostgresql_confirmed-post_notify_demoted_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold] "msPostgresql_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "msPostgresql_confirmed-post_notify_stopped_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold] "msPostgresql_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_confirmed-pre_notify_demote_0" -> "msPostgresql_demote_0" [ style = bold] "msPostgresql_confirmed-pre_notify_demote_0" -> "msPostgresql_post_notify_demoted_0" [ style = bold] "msPostgresql_confirmed-pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_confirmed-pre_notify_stop_0" -> "msPostgresql_post_notify_stopped_0" [ style = bold] "msPostgresql_confirmed-pre_notify_stop_0" -> "msPostgresql_stop_0" [ style = bold] "msPostgresql_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_demote_0" -> "msPostgresql_demoted_0" [ style = bold] "msPostgresql_demote_0" -> "pgsql_demote_0 pgsr02" [ style = bold] "msPostgresql_demote_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_demoted_0" -> "msPostgresql_post_notify_demoted_0" [ style = bold] "msPostgresql_demoted_0" -> "msPostgresql_stop_0" [ style = bold] "msPostgresql_demoted_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_post_notify_demoted_0" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold] "msPostgresql_post_notify_demoted_0" -> "pgsql_post_notify_demoted_0 pgsr01" [ style = bold] "msPostgresql_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_post_notify_stopped_0" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold] "msPostgresql_post_notify_stopped_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold] "msPostgresql_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_pre_notify_demote_0" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold] "msPostgresql_pre_notify_demote_0" -> "pgsql_pre_notify_demote_0 pgsr01" [ style = bold] "msPostgresql_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_pre_notify_stop_0" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold] "msPostgresql_pre_notify_stop_0" -> "pgsql_pre_notify_stop_0 pgsr01" [ style = bold] "msPostgresql_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_stop_0" -> "msPostgresql_stopped_0" [ style = bold] "msPostgresql_stop_0" -> "pgsql_stop_0 pgsr02" [ style = bold] "msPostgresql_stop_0" [ style=bold color="green" fontcolor="orange"] "msPostgresql_stopped_0" -> "msPostgresql_post_notify_stopped_0" [ style = bold] "msPostgresql_stopped_0" [ style=bold color="green" fontcolor="orange"] "pgsql_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "pgsql_confirmed-post_notify_stonith_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold] "pgsql_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "pgsql_demote_0 pgsr02" -> "msPostgresql_demoted_0" [ style = bold] "pgsql_demote_0 pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold] "pgsql_demote_0 pgsr02" [ style=bold color="green" fontcolor="orange"] "pgsql_monitor_9000 pgsr01" [ style=bold color="green" fontcolor="black"] "pgsql_post_notify_demoted_0 pgsr01" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold] "pgsql_post_notify_demoted_0 pgsr01" [ style=bold color="green" fontcolor="black"] "pgsql_post_notify_stonith_0 pgsr01" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold] "pgsql_post_notify_stonith_0 pgsr01" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold] "pgsql_post_notify_stonith_0 pgsr01" [ style=bold color="green" fontcolor="black"] "pgsql_post_notify_stonith_0" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold] "pgsql_post_notify_stonith_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold] "pgsql_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "pgsql_pre_notify_demote_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold] "pgsql_pre_notify_demote_0 pgsr01" [ style=bold color="green" fontcolor="black"] "pgsql_pre_notify_stop_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold] "pgsql_pre_notify_stop_0 pgsr01" [ style=bold color="green" fontcolor="black"] "pgsql_stop_0 pgsr02" -> "all_stopped" [ style = bold] "pgsql_stop_0 pgsr02" -> "msPostgresql_stopped_0" [ style = bold] "pgsql_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"] "pgsr01_monitor_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] "pgsr02_monitor_0 bl460g8n3" -> "pgsr02_stop_0 bl460g8n4" [ style = bold] "pgsr02_monitor_0 bl460g8n3" [ style=bold color="green" fontcolor="black"] "pgsr02_stop_0 bl460g8n4" -> "all_stopped" [ style = bold] "pgsr02_stop_0 bl460g8n4" -> "prmDB2_stop_0 bl460g8n4" [ style = bold] "pgsr02_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] "prmDB2_stop_0 bl460g8n4" -> "all_stopped" [ style = bold] "prmDB2_stop_0 bl460g8n4" -> "stonith 'off' pgsr02" [ style = bold] "prmDB2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] "prmStonith1-2_monitor_3600000 bl460g8n4" [ style=bold color="green" fontcolor="black"] "prmStonith1-2_start_0 bl460g8n4" -> "grpStonith1_running_0" [ style = bold] "prmStonith1-2_start_0 bl460g8n4" -> "prmStonith1-2_monitor_3600000 bl460g8n4" [ style = bold] "prmStonith1-2_start_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] "prmStonith1-2_stop_0 bl460g8n4" -> "all_stopped" [ style = bold] "prmStonith1-2_stop_0 bl460g8n4" -> "grpStonith1_stopped_0" [ style = bold] "prmStonith1-2_stop_0 bl460g8n4" -> "prmStonith1-2_start_0 bl460g8n4" [ style = bold] "prmStonith1-2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] "prmStonith2-2_monitor_3600000 bl460g8n3" [ style=bold color="green" fontcolor="black"] "prmStonith2-2_start_0 bl460g8n3" -> "grpStonith2_running_0" [ style = bold] "prmStonith2-2_start_0 bl460g8n3" -> "prmStonith2-2_monitor_3600000 bl460g8n3" [ style = bold] "prmStonith2-2_start_0 bl460g8n3" [ style=bold color="green" fontcolor="black"] "prmStonith2-2_stop_0 bl460g8n3" -> "all_stopped" [ style = bold] "prmStonith2-2_stop_0 bl460g8n3" -> "grpStonith2_stopped_0" [ style = bold] "prmStonith2-2_stop_0 bl460g8n3" -> "prmStonith2-2_start_0 bl460g8n3" [ style = bold] "prmStonith2-2_stop_0 bl460g8n3" [ style=bold color="green" fontcolor="black"] "stonith 'off' pgsr02" -> "master-group_stop_0" [ style = bold] "stonith 'off' pgsr02" -> "msPostgresql_stop_0" [ style = bold] "stonith 'off' pgsr02" -> "pgsql_demote_0 pgsr02" [ style = bold] "stonith 'off' pgsr02" -> "pgsql_post_notify_stonith_0" [ style = bold] "stonith 'off' pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold] "stonith 'off' pgsr02" -> "stonith_complete" [ style = bold] "stonith 'off' pgsr02" -> "vip-master_stop_0 pgsr02" [ style = bold] "stonith 'off' pgsr02" -> "vip-rep_stop_0 pgsr02" [ style = bold] "stonith 'off' pgsr02" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "vip-master_start_0 pgsr01" [ style = bold] "stonith_complete" -> "vip-rep_start_0 pgsr01" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] +"vip-master_monitor_0 pgsr01" -> "vip-master_start_0 pgsr01" [ style = bold] +"vip-master_monitor_0 pgsr01" -> "vip-master_stop_0 pgsr02" [ style = bold] +"vip-master_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-master_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-master_start_0 pgsr01" -> "master-group_running_0" [ style = bold] "vip-master_start_0 pgsr01" -> "vip-master_monitor_10000 pgsr01" [ style = bold] "vip-master_start_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold] "vip-master_start_0 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-master_stop_0 pgsr02" -> "all_stopped" [ style = bold] "vip-master_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold] "vip-master_stop_0 pgsr02" -> "vip-master_start_0 pgsr01" [ style = bold] "vip-master_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"] +"vip-rep_monitor_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold] +"vip-rep_monitor_0 pgsr01" -> "vip-rep_stop_0 pgsr02" [ style = bold] +"vip-rep_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-rep_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-rep_start_0 pgsr01" -> "master-group_running_0" [ style = bold] "vip-rep_start_0 pgsr01" -> "vip-rep_monitor_10000 pgsr01" [ style = bold] "vip-rep_start_0 pgsr01" [ style=bold color="green" fontcolor="black"] "vip-rep_stop_0 pgsr02" -> "all_stopped" [ style = bold] "vip-rep_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold] "vip-rep_stop_0 pgsr02" -> "vip-master_stop_0 pgsr02" [ style = bold] "vip-rep_stop_0 pgsr02" -> "vip-rep_start_0 pgsr01" [ style = bold] "vip-rep_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp index 2ec83c092d..47c369a93d 100644 --- a/pengine/test10/bug-cl-5247.exp +++ b/pengine/test10/bug-cl-5247.exp @@ -1,772 +1,802 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + + + + - + - + - + + + + + + + + + + - + - + - + - + + + + - + - + - + - + - + - + + + + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary index 3e5511db9c..f15ef2e70a 100644 --- a/pengine/test10/bug-cl-5247.summary +++ b/pengine/test10/bug-cl-5247.summary @@ -1,101 +1,103 @@ Using the original execution date of: 2015-08-12 02:53:40Z Current cluster status: Online: [ bl460g8n3 bl460g8n4 ] Containers: [ pgsr01:prmDB1 ] prmDB1 (ocf::heartbeat:VirtualDomain): Started bl460g8n3 prmDB2 (ocf::heartbeat:VirtualDomain): FAILED bl460g8n4 Resource Group: grpStonith1 prmStonith1-2 (stonith:external/ipmi): Started bl460g8n4 Resource Group: grpStonith2 prmStonith2-2 (stonith:external/ipmi): Started bl460g8n3 Resource Group: master-group vip-master (ocf::heartbeat:Dummy): FAILED pgsr02 vip-rep (ocf::heartbeat:Dummy): FAILED pgsr02 Master/Slave Set: msPostgresql [pgsql] Masters: [ pgsr01 ] Stopped: [ bl460g8n3 bl460g8n4 ] Transition Summary: * Fence (off) pgsr02 (resource: prmDB2) 'guest is unclean' * Stop prmDB2 (bl460g8n4) due to node availability * Restart prmStonith1-2 ( bl460g8n4 ) * Restart prmStonith2-2 ( bl460g8n3 ) * Recover vip-master ( pgsr02 -> pgsr01 ) * Recover vip-rep ( pgsr02 -> pgsr01 ) * Stop pgsql:0 ( Master pgsr02 ) due to node availability * Stop pgsr02 (bl460g8n4) Executing cluster transition: * Pseudo action: grpStonith1_stop_0 * Resource action: prmStonith1-2 stop on bl460g8n4 * Pseudo action: grpStonith2_stop_0 * Resource action: prmStonith2-2 stop on bl460g8n3 + * Resource action: vip-master monitor on pgsr01 + * Resource action: vip-rep monitor on pgsr01 * Pseudo action: msPostgresql_pre_notify_demote_0 * Resource action: pgsr01 monitor on bl460g8n4 * Resource action: pgsr02 monitor on bl460g8n3 * Pseudo action: grpStonith1_stopped_0 * Pseudo action: grpStonith1_start_0 * Pseudo action: grpStonith2_stopped_0 * Pseudo action: grpStonith2_start_0 * Resource action: pgsql notify on pgsr01 * Pseudo action: msPostgresql_confirmed-pre_notify_demote_0 * Pseudo action: msPostgresql_demote_0 * Resource action: pgsr02 stop on bl460g8n4 * Resource action: prmDB2 stop on bl460g8n4 * Pseudo action: stonith-pgsr02-off on pgsr02 * Pseudo action: stonith_complete * Pseudo action: pgsql_post_notify_stop_0 * Pseudo action: pgsql_demote_0 * Pseudo action: msPostgresql_demoted_0 * Pseudo action: msPostgresql_post_notify_demoted_0 * Resource action: pgsql notify on pgsr01 * Pseudo action: msPostgresql_confirmed-post_notify_demoted_0 * Pseudo action: msPostgresql_pre_notify_stop_0 * Pseudo action: master-group_stop_0 * Pseudo action: vip-rep_stop_0 * Resource action: pgsql notify on pgsr01 * Pseudo action: msPostgresql_confirmed-pre_notify_stop_0 * Pseudo action: msPostgresql_stop_0 * Pseudo action: vip-master_stop_0 * Pseudo action: pgsql_stop_0 * Pseudo action: msPostgresql_stopped_0 * Pseudo action: master-group_stopped_0 * Pseudo action: master-group_start_0 * Resource action: vip-master start on pgsr01 * Resource action: vip-rep start on pgsr01 * Pseudo action: msPostgresql_post_notify_stopped_0 * Pseudo action: master-group_running_0 * Resource action: vip-master monitor=10000 on pgsr01 * Resource action: vip-rep monitor=10000 on pgsr01 * Resource action: pgsql notify on pgsr01 * Pseudo action: msPostgresql_confirmed-post_notify_stopped_0 * Pseudo action: pgsql_notified_0 * Resource action: pgsql monitor=9000 on pgsr01 * Pseudo action: all_stopped * Resource action: prmStonith1-2 start on bl460g8n4 * Resource action: prmStonith1-2 monitor=3600000 on bl460g8n4 * Resource action: prmStonith2-2 start on bl460g8n3 * Resource action: prmStonith2-2 monitor=3600000 on bl460g8n3 * Pseudo action: grpStonith1_running_0 * Pseudo action: grpStonith2_running_0 Using the original execution date of: 2015-08-12 02:53:40Z Revised cluster status: Online: [ bl460g8n3 bl460g8n4 ] Containers: [ pgsr01:prmDB1 ] prmDB1 (ocf::heartbeat:VirtualDomain): Started bl460g8n3 prmDB2 (ocf::heartbeat:VirtualDomain): FAILED Resource Group: grpStonith1 prmStonith1-2 (stonith:external/ipmi): Started bl460g8n4 Resource Group: grpStonith2 prmStonith2-2 (stonith:external/ipmi): Started bl460g8n3 Resource Group: master-group vip-master (ocf::heartbeat:Dummy): FAILED[ pgsr01 pgsr02 ] vip-rep (ocf::heartbeat:Dummy): FAILED[ pgsr01 pgsr02 ] Master/Slave Set: msPostgresql [pgsql] Masters: [ pgsr01 ] Stopped: [ bl460g8n3 bl460g8n4 ] diff --git a/pengine/test10/bug-rh-1097457.dot b/pengine/test10/bug-rh-1097457.dot index e74c8fbdc7..7791e0627a 100644 --- a/pengine/test10/bug-rh-1097457.dot +++ b/pengine/test10/bug-rh-1097457.dot @@ -1,100 +1,132 @@ digraph "g" { +"FAKE1-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE2-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE3-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] +"FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] +"FAKE4-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] +"FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] +"FAKE4-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_start_0 lamaVM2" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] "FAKE4-IP_start_0 lamaVM2" -> "lamaVM2-G4_running_0" [ style = bold] "FAKE4-IP_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_stop_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] +"FAKE4_monitor_0 lamaVM1" -> "FAKE4_start_0 lamaVM2" [ style = bold] +"FAKE4_monitor_0 lamaVM1" -> "FAKE4_stop_0 lamaVM2" [ style = bold] +"FAKE4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FAKE4_monitor_0 lamaVM3" -> "FAKE4_start_0 lamaVM2" [ style = bold] +"FAKE4_monitor_0 lamaVM3" -> "FAKE4_stop_0 lamaVM2" [ style = bold] +"FAKE4_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE4_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4_start_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4_start_0 lamaVM2" -> "FAKE4_monitor_30000 lamaVM2" [ style = bold] "FAKE4_start_0 lamaVM2" -> "lamaVM2-G4_running_0" [ style = bold] "FAKE4_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4_stop_0 lamaVM2" -> "FAKE4_start_0 lamaVM2" [ style = bold] "FAKE4_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE4_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] "FAKE4_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] +"FAKE5-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FAKE5-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FAKE5_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FAKE6-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FAKE6-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE6-clone_running_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_start_0" -> "FAKE6-clone_running_0" [ style = bold] "FAKE6-clone_start_0" -> "FAKE6_start_0 lamaVM2" [ style = bold] "FAKE6-clone_start_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_stop_0" -> "FAKE6-clone_stopped_0" [ style = bold] "FAKE6-clone_stop_0" -> "FAKE6_stop_0 lamaVM2" [ style = bold] "FAKE6-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_stopped_0" -> "FAKE6-clone_start_0" [ style = bold] "FAKE6-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "FAKE6_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE6_start_0 lamaVM2" -> "FAKE6-clone_running_0" [ style = bold] "FAKE6_start_0 lamaVM2" -> "FAKE6_monitor_30000 lamaVM2" [ style = bold] "FAKE6_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE6_stop_0 lamaVM2" -> "FAKE6-clone_stopped_0" [ style = bold] "FAKE6_stop_0 lamaVM2" -> "FAKE6_start_0 lamaVM2" [ style = bold] "FAKE6_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE6_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] +"FSlun1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FSlun2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] +"FSlun3_monitor_0 lamaVM1" -> "FSlun3_start_0 lama2" [ style = bold] +"FSlun3_monitor_0 lamaVM1" -> "FSlun3_stop_0 lamaVM2" [ style = bold] +"FSlun3_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] +"FSlun3_monitor_0 lamaVM3" -> "FSlun3_start_0 lama2" [ style = bold] +"FSlun3_monitor_0 lamaVM3" -> "FSlun3_stop_0 lamaVM2" [ style = bold] +"FSlun3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_10000 lama2" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_10000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FSlun3_start_0 lama2" -> "FSlun3_monitor_10000 lama2" [ style = bold] "FSlun3_start_0 lama2" -> "lamaVM2-G4_start_0" [ style = bold] "FSlun3_start_0 lama2" [ style=bold color="green" fontcolor="black"] "FSlun3_stop_0 lamaVM2" -> "FSlun3_start_0 lama2" [ style = bold] "FSlun3_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FSlun3_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] +"FSlun4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "VM2_monitor_10000 lama3" [ style=bold color="green" fontcolor="black"] "VM2_start_0 lama3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FAKE4_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FAKE6_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FSlun3_start_0 lama2" [ style = bold] "VM2_start_0 lama3" -> "VM2_monitor_10000 lama3" [ style = bold] "VM2_start_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] "VM2_start_0 lama3" [ style=bold color="green" fontcolor="black"] "VM2_stop_0 lama3" -> "VM2_start_0 lama3" [ style = bold] "VM2_stop_0 lama3" -> "all_stopped" [ style = bold] "VM2_stop_0 lama3" -> "stonith 'reboot' lamaVM2" [ style = bold] "VM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_running_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_start_0" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "lamaVM2-G4_start_0" -> "FAKE4_start_0 lamaVM2" [ style = bold] "lamaVM2-G4_start_0" -> "lamaVM2-G4_running_0" [ style = bold] "lamaVM2-G4_start_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_stop_0" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stop_0" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stop_0" -> "lamaVM2-G4_stopped_0" [ style = bold] "lamaVM2-G4_stop_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_stopped_0" -> "FSlun3_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stopped_0" -> "lamaVM2-G4_start_0" [ style = bold] "lamaVM2-G4_stopped_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2_monitor_30000 lama3" [ style=bold color="green" fontcolor="black"] "lamaVM2_start_0 lama3" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE6_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE6_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FSlun3_monitor_10000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "lamaVM2_monitor_30000 lama3" [ style = bold] "lamaVM2_start_0 lama3" [ style=bold color="green" fontcolor="black"] "lamaVM2_stop_0 lama3" -> "VM2_stop_0 lama3" [ style = bold] "lamaVM2_stop_0 lama3" -> "all_stopped" [ style = bold] "lamaVM2_stop_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] "lamaVM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lamaVM2" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE6-clone_stop_0" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE6_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FSlun3_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "lamaVM2-G4_stop_0" [ style = bold] "stonith 'reboot' lamaVM2" -> "stonith_complete" [ style = bold] "stonith 'reboot' lamaVM2" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FAKE4_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FAKE6_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FSlun3_start_0 lama2" [ style = bold] "stonith_complete" -> "VM2_start_0 lama3" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "lamaVM2_start_0 lama3" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp index eb9b225bd1..33c13bdc36 100644 --- a/pengine/test10/bug-rh-1097457.exp +++ b/pengine/test10/bug-rh-1097457.exp @@ -1,496 +1,712 @@ - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - + - + + + + + + + - + - + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - + - + + + + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + + + + - + - + - + - + - + + + + - + - + - + - + + + + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary index c50df5fb45..e23c6adc25 100644 --- a/pengine/test10/bug-rh-1097457.summary +++ b/pengine/test10/bug-rh-1097457.summary @@ -1,105 +1,125 @@ 2 of 26 resources DISABLED and 0 BLOCKED from being started due to failures Current cluster status: Online: [ lama2 lama3 ] Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 VM1 (ocf::heartbeat:VirtualDomain): Started lama2 FSlun1 (ocf::heartbeat:Filesystem): Started lamaVM1 FSlun2 (ocf::heartbeat:Filesystem): Started lamaVM1 VM2 (ocf::heartbeat:VirtualDomain): FAILED lama3 VM3 (ocf::heartbeat:VirtualDomain): Started lama3 FSlun3 (ocf::heartbeat:Filesystem): FAILED lamaVM2 FSlun4 (ocf::heartbeat:Filesystem): Started lamaVM3 FAKE5-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE6-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE5 (ocf::heartbeat:Dummy): Started lamaVM3 Resource Group: lamaVM1-G1 FAKE1 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE1-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G2 FAKE2 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE2-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G3 FAKE3 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE3-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM2-G4 FAKE4 (ocf::heartbeat:Dummy): Started lamaVM2 FAKE4-IP (ocf::heartbeat:IPaddr2): Started lamaVM2 Clone Set: FAKE6-clone [FAKE6] Started: [ lamaVM1 lamaVM2 lamaVM3 ] Transition Summary: * Fence (reboot) lamaVM2 (resource: VM2) 'guest is unclean' * Recover VM2 ( lama3 ) * Recover FSlun3 ( lamaVM2 -> lama2 ) * Restart FAKE4 ( lamaVM2 ) due to required VM2 start * Restart FAKE4-IP ( lamaVM2 ) due to required VM2 start * Restart FAKE6:2 ( lamaVM2 ) due to required VM2 start * Restart lamaVM2 ( lama3 ) due to required VM2 start Executing cluster transition: + * Resource action: FSlun1 monitor on lamaVM3 + * Resource action: FSlun2 monitor on lamaVM3 + * Resource action: FSlun3 monitor on lamaVM3 + * Resource action: FSlun3 monitor on lamaVM1 + * Resource action: FSlun4 monitor on lamaVM1 + * Resource action: FAKE5-IP monitor on lamaVM3 + * Resource action: FAKE5-IP monitor on lamaVM1 + * Resource action: FAKE6-IP monitor on lamaVM3 + * Resource action: FAKE6-IP monitor on lamaVM1 + * Resource action: FAKE5 monitor on lamaVM1 + * Resource action: FAKE1 monitor on lamaVM3 + * Resource action: FAKE1-IP monitor on lamaVM3 + * Resource action: FAKE2 monitor on lamaVM3 + * Resource action: FAKE2-IP monitor on lamaVM3 + * Resource action: FAKE3 monitor on lamaVM3 + * Resource action: FAKE3-IP monitor on lamaVM3 + * Resource action: FAKE4 monitor on lamaVM3 + * Resource action: FAKE4 monitor on lamaVM1 + * Resource action: FAKE4-IP monitor on lamaVM3 + * Resource action: FAKE4-IP monitor on lamaVM1 * Resource action: lamaVM2 stop on lama3 * Resource action: VM2 stop on lama3 * Pseudo action: stonith-lamaVM2-reboot on lamaVM2 * Pseudo action: stonith_complete * Resource action: VM2 start on lama3 * Resource action: VM2 monitor=10000 on lama3 * Pseudo action: lamaVM2-G4_stop_0 * Pseudo action: FAKE4-IP_stop_0 * Pseudo action: FAKE6-clone_stop_0 * Resource action: lamaVM2 start on lama3 * Resource action: lamaVM2 monitor=30000 on lama3 * Resource action: FSlun3 monitor=10000 on lamaVM2 * Pseudo action: FAKE4_stop_0 * Pseudo action: FAKE6_stop_0 * Pseudo action: FAKE6-clone_stopped_0 * Pseudo action: FAKE6-clone_start_0 * Pseudo action: lamaVM2-G4_stopped_0 * Resource action: FAKE6 start on lamaVM2 * Resource action: FAKE6 monitor=30000 on lamaVM2 * Pseudo action: FAKE6-clone_running_0 * Pseudo action: FSlun3_stop_0 * Pseudo action: all_stopped * Resource action: FSlun3 start on lama2 * Pseudo action: lamaVM2-G4_start_0 * Resource action: FAKE4 start on lamaVM2 * Resource action: FAKE4 monitor=30000 on lamaVM2 * Resource action: FAKE4-IP start on lamaVM2 * Resource action: FAKE4-IP monitor=30000 on lamaVM2 * Resource action: FSlun3 monitor=10000 on lama2 * Pseudo action: lamaVM2-G4_running_0 Revised cluster status: Online: [ lama2 lama3 ] Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 VM1 (ocf::heartbeat:VirtualDomain): Started lama2 FSlun1 (ocf::heartbeat:Filesystem): Started lamaVM1 FSlun2 (ocf::heartbeat:Filesystem): Started lamaVM1 VM2 (ocf::heartbeat:VirtualDomain): FAILED lama3 VM3 (ocf::heartbeat:VirtualDomain): Started lama3 FSlun3 (ocf::heartbeat:Filesystem): FAILED [ lama2 lamaVM2 ] FSlun4 (ocf::heartbeat:Filesystem): Started lamaVM3 FAKE5-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE6-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE5 (ocf::heartbeat:Dummy): Started lamaVM3 Resource Group: lamaVM1-G1 FAKE1 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE1-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G2 FAKE2 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE2-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G3 FAKE3 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE3-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM2-G4 FAKE4 (ocf::heartbeat:Dummy): Started lamaVM2 FAKE4-IP (ocf::heartbeat:IPaddr2): Started lamaVM2 Clone Set: FAKE6-clone [FAKE6] Started: [ lamaVM1 lamaVM2 lamaVM3 ] diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot index d58e8096d5..163be1f125 100644 --- a/pengine/test10/bundle-order-partial-start-2.dot +++ b/pengine/test10/bundle-order-partial-start-2.dot @@ -1,101 +1,103 @@ digraph "g" { "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] "galera-bundle-0_stop_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] "galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] "galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] "galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] "galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] "galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] "galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] "galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] "rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] +"rabbitmq:0_monitor_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] +"rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] "redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] "redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] "redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp index f600983462..087ed6cc6e 100644 --- a/pengine/test10/bundle-order-partial-start-2.exp +++ b/pengine/test10/bundle-order-partial-start-2.exp @@ -1,566 +1,578 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary index bf14db0191..50933a2019 100644 --- a/pengine/test10/bundle-order-partial-start-2.summary +++ b/pengine/test10/bundle-order-partial-start-2.summary @@ -1,95 +1,96 @@ Current cluster status: Online: [ undercloud ] Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Stopped undercloud Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud Transition Summary: * Start rabbitmq:0 (rabbitmq-bundle-0) * Restart galera-bundle-docker-0 ( undercloud ) due to required haproxy-bundle running * Restart galera-bundle-0 ( undercloud ) due to required galera-bundle-docker-0 start * Start galera:0 (galera-bundle-0) * Promote redis:0 (Slave -> Master redis-bundle-0) * Start haproxy-bundle-docker-0 (undercloud) Executing cluster transition: + * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 * Resource action: galera-bundle-0 stop on undercloud * Pseudo action: redis-bundle-master_pre_notify_promote_0 * Resource action: haproxy-bundle-docker-0 monitor on undercloud * Pseudo action: haproxy-bundle_start_0 * Pseudo action: redis-bundle_promote_0 * Pseudo action: rabbitmq-bundle_start_0 * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 * Pseudo action: rabbitmq-bundle-clone_start_0 * Resource action: galera-bundle-docker-0 stop on undercloud * Resource action: redis notify on redis-bundle-0 * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 * Pseudo action: redis-bundle-master_promote_0 * Resource action: haproxy-bundle-docker-0 start on undercloud * Pseudo action: haproxy-bundle_running_0 * Pseudo action: all_stopped * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_running_0 * Resource action: redis promote on redis-bundle-0 * Pseudo action: redis-bundle-master_promoted_0 * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 * Pseudo action: redis-bundle-master_post_notify_promoted_0 * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 * Resource action: redis notify on redis-bundle-0 * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 * Pseudo action: redis-bundle_promoted_0 * Pseudo action: rabbitmq-bundle_running_0 * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 * Resource action: redis monitor=20000 on redis-bundle-0 * Pseudo action: galera-bundle_start_0 * Pseudo action: galera-bundle-master_start_0 * Resource action: galera-bundle-docker-0 start on undercloud * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud * Resource action: galera-bundle-0 start on undercloud * Resource action: galera-bundle-0 monitor=60000 on undercloud * Resource action: galera:0 start on galera-bundle-0 * Pseudo action: galera-bundle-master_running_0 * Pseudo action: galera-bundle_running_0 * Resource action: galera:0 monitor=30000 on galera-bundle-0 * Resource action: galera:0 monitor=20000 on galera-bundle-0 Revised cluster status: Online: [ undercloud ] Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot index 796749df4e..958e7f4d17 100644 --- a/pengine/test10/bundle-order-partial-start.dot +++ b/pengine/test10/bundle-order-partial-start.dot @@ -1,95 +1,97 @@ digraph "g" { "galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] "galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] "galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] "galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] "galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] "galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] "galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] "galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] "haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] "rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] +"rabbitmq:0_monitor_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] +"rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] "redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] "redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] "redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp index d48fccf766..8aad157fef 100644 --- a/pengine/test10/bundle-order-partial-start.exp +++ b/pengine/test10/bundle-order-partial-start.exp @@ -1,532 +1,544 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary index 4e02e88a67..9045545dd4 100644 --- a/pengine/test10/bundle-order-partial-start.summary +++ b/pengine/test10/bundle-order-partial-start.summary @@ -1,93 +1,94 @@ Current cluster status: Online: [ undercloud ] Containers: [ rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Stopped Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud Transition Summary: * Start rabbitmq:0 (rabbitmq-bundle-0) * Start galera-bundle-docker-0 (undercloud) * Start galera-bundle-0 (undercloud) * Start galera:0 (galera-bundle-0) * Promote redis:0 (Slave -> Master redis-bundle-0) * Start haproxy-bundle-docker-0 (undercloud) Executing cluster transition: + * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 * Resource action: galera-bundle-docker-0 monitor on undercloud * Pseudo action: redis-bundle-master_pre_notify_promote_0 * Resource action: haproxy-bundle-docker-0 monitor on undercloud * Pseudo action: haproxy-bundle_start_0 * Pseudo action: redis-bundle_promote_0 * Pseudo action: rabbitmq-bundle_start_0 * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 * Pseudo action: rabbitmq-bundle-clone_start_0 * Resource action: redis notify on redis-bundle-0 * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 * Pseudo action: redis-bundle-master_promote_0 * Resource action: haproxy-bundle-docker-0 start on undercloud * Pseudo action: haproxy-bundle_running_0 * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_running_0 * Resource action: redis promote on redis-bundle-0 * Pseudo action: redis-bundle-master_promoted_0 * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 * Pseudo action: redis-bundle-master_post_notify_promoted_0 * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 * Resource action: redis notify on redis-bundle-0 * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 * Pseudo action: redis-bundle_promoted_0 * Pseudo action: rabbitmq-bundle_running_0 * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 * Resource action: redis monitor=20000 on redis-bundle-0 * Pseudo action: galera-bundle_start_0 * Pseudo action: galera-bundle-master_start_0 * Resource action: galera-bundle-docker-0 start on undercloud * Resource action: galera-bundle-0 start on undercloud * Resource action: galera:0 start on galera-bundle-0 * Pseudo action: galera-bundle-master_running_0 * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud * Resource action: galera-bundle-0 monitor=60000 on undercloud * Pseudo action: galera-bundle_running_0 * Resource action: galera:0 monitor=30000 on galera-bundle-0 * Resource action: galera:0 monitor=20000 on galera-bundle-0 Revised cluster status: Online: [ undercloud ] Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp index 03b064a639..ec1053c162 100644 --- a/pengine/test10/bundle-order-startup.exp +++ b/pengine/test10/bundle-order-startup.exp @@ -1,975 +1,975 @@ - + - + - + - + diff --git a/pengine/test10/container-is-remote-node.dot b/pengine/test10/container-is-remote-node.dot index d8f1c9f22b..ebb0793da5 100644 --- a/pengine/test10/container-is-remote-node.dot +++ b/pengine/test10/container-is-remote-node.dot @@ -1,2 +1,6 @@ digraph "g" { +"clvmd_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] +"dlm_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] +"gfs2-lv_1_1_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] +"gfs2-lv_1_2_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/container-is-remote-node.exp b/pengine/test10/container-is-remote-node.exp index 56e315ff01..d656dd277d 100644 --- a/pengine/test10/container-is-remote-node.exp +++ b/pengine/test10/container-is-remote-node.exp @@ -1 +1,38 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pengine/test10/container-is-remote-node.summary b/pengine/test10/container-is-remote-node.summary index 6ed05267af..f5c78ce503 100644 --- a/pengine/test10/container-is-remote-node.summary +++ b/pengine/test10/container-is-remote-node.summary @@ -1,52 +1,56 @@ 3 of 19 resources DISABLED and 0 BLOCKED from being started due to failures Current cluster status: Online: [ lama2 lama3 ] Containers: [ RNVM1:VM1 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 Clone Set: dlm-clone [dlm] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: clvmd-clone [clvmd] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: gfs2-lv_1_1-clone [gfs2-lv_1_1] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: gfs2-lv_1_2-clone [gfs2-lv_1_2] Stopped (disabled): [ RNVM1 lama2 lama3 ] VM1 (ocf::heartbeat:VirtualDomain): Started lama2 Resource Group: RES1 FSdata1 (ocf::heartbeat:Filesystem): Started RNVM1 RES1-IP (ocf::heartbeat:IPaddr2): Started RNVM1 res-rsyslog (ocf::heartbeat:rsyslog.test): Started RNVM1 Transition Summary: Executing cluster transition: + * Resource action: dlm monitor on RNVM1 + * Resource action: clvmd monitor on RNVM1 + * Resource action: gfs2-lv_1_1 monitor on RNVM1 + * Resource action: gfs2-lv_1_2 monitor on RNVM1 Revised cluster status: Online: [ lama2 lama3 ] Containers: [ RNVM1:VM1 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 Clone Set: dlm-clone [dlm] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: clvmd-clone [clvmd] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: gfs2-lv_1_1-clone [gfs2-lv_1_1] Started: [ lama2 lama3 ] Stopped: [ RNVM1 ] Clone Set: gfs2-lv_1_2-clone [gfs2-lv_1_2] Stopped (disabled): [ RNVM1 lama2 lama3 ] VM1 (ocf::heartbeat:VirtualDomain): Started lama2 Resource Group: RES1 FSdata1 (ocf::heartbeat:Filesystem): Started RNVM1 RES1-IP (ocf::heartbeat:IPaddr2): Started RNVM1 res-rsyslog (ocf::heartbeat:rsyslog.test): Started RNVM1 diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot index 04152d1636..83f38c4191 100644 --- a/pengine/test10/guest-node-host-dies.dot +++ b/pengine/test10/guest-node-host-dies.dot @@ -1,137 +1,139 @@ digraph "g" { "Fencing_monitor_120000 rhel7-4" [ style=bold color="green" fontcolor="black"] "Fencing_start_0 rhel7-4" -> "Fencing_monitor_120000 rhel7-4" [ style = bold] "Fencing_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-4" [ style = bold] "Fencing_stop_0 rhel7-4" -> "all_stopped" [ style = bold] "Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "all_stopped" -> "Fencing_start_0 rhel7-4" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] "container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] "container1_stop_0 rhel7-1" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] "container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] "container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold] "container2_stop_0 rhel7-1" -> "stonith 'reboot' lxc2" [ style = bold] "container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] "lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] "lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold] "lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] "lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] "lxc1_monitor_0 rhel7-3" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-3" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-4" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-4" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-5" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-5" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc1_monitor_30000 rhel7-2" [ style = bold] "lxc1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] "lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] +"lxc1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "lxc2_monitor_0 rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-2" -> "lxc2_stop_0 rhel7-1" [ style = bold] "lxc2_monitor_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-4" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-4" -> "lxc2_stop_0 rhel7-1" [ style = bold] "lxc2_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-5" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-5" -> "lxc2_stop_0 rhel7-1" [ style = bold] "lxc2_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold] "lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] "lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold] +"lxc2_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"] "rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold] "rsc_rhel7-1_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "rsc_rhel7-1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "rsc_rhel7-1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "rsc_rhel7-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] "stonith 'reboot' lxc2" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "lxc2_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "rsc_rhel7-1_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 rhel7-2" [ style = bold] "stonith_complete" -> "container2_start_0 rhel7-3" [ style = bold] "stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold] "stonith_complete" -> "lxc1_start_0 rhel7-2" [ style = bold] "stonith_complete" -> "lxc2_start_0 rhel7-3" [ style = bold] "stonith_complete" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp index cd8d81a748..01fe67861a 100644 --- a/pengine/test10/guest-node-host-dies.exp +++ b/pengine/test10/guest-node-host-dies.exp @@ -1,682 +1,688 @@ + + + + + + diff --git a/pengine/test10/whitebox-asymmetric.dot b/pengine/test10/whitebox-asymmetric.dot index 896f66a790..da2f3e32ba 100644 --- a/pengine/test10/whitebox-asymmetric.dot +++ b/pengine/test10/whitebox-asymmetric.dot @@ -1,10 +1,14 @@ digraph "g" { "18node2_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] "18node2_start_0 18builder" -> "18node2_monitor_30000 18builder" [ style = bold] "18node2_start_0 18builder" -> "nfs_mount_monitor_10000 18node2" [ style = bold] "18node2_start_0 18builder" -> "nfs_mount_start_0 18node2" [ style = bold] +"18node2_start_0 18builder" -> "vg_tags_dup_monitor_0 18node2" [ style = bold] +"18node2_start_0 18builder" -> "webserver_monitor_0 18node2" [ style = bold] "18node2_start_0 18builder" [ style=bold color="green" fontcolor="black"] "nfs_mount_monitor_10000 18node2" [ style=bold color="green" fontcolor="black"] "nfs_mount_start_0 18node2" -> "nfs_mount_monitor_10000 18node2" [ style = bold] "nfs_mount_start_0 18node2" [ style=bold color="green" fontcolor="black"] +"vg_tags_dup_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] +"webserver_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-asymmetric.exp b/pengine/test10/whitebox-asymmetric.exp index 1178fb622b..db714439e0 100644 --- a/pengine/test10/whitebox-asymmetric.exp +++ b/pengine/test10/whitebox-asymmetric.exp @@ -1,53 +1,79 @@ - + + + + + + + + + + + + + + - + - + - + - + - + - + - + + + + + + + + + + + + + + - + - + - + diff --git a/pengine/test10/whitebox-asymmetric.summary b/pengine/test10/whitebox-asymmetric.summary index 1c5011de9f..8da999649f 100644 --- a/pengine/test10/whitebox-asymmetric.summary +++ b/pengine/test10/whitebox-asymmetric.summary @@ -1,35 +1,37 @@ 2 of 7 resources DISABLED and 0 BLOCKED from being started due to failures Current cluster status: Online: [ 18builder ] fence_false (stonith:fence_false): Stopped container2 (ocf::pacemaker:Dummy): Started 18builder webserver (ocf::pacemaker:Dummy): Stopped nfs_mount (ocf::pacemaker:Dummy): Stopped Resource Group: mygroup vg_tags (ocf::heartbeat:LVM): Stopped ( disabled ) vg_tags_dup (ocf::heartbeat:LVM): Stopped Transition Summary: * Start nfs_mount (18node2) * Start 18node2 (18builder) Executing cluster transition: * Resource action: 18node2 start on 18builder + * Resource action: webserver monitor on 18node2 * Resource action: nfs_mount start on 18node2 + * Resource action: vg_tags_dup monitor on 18node2 * Resource action: 18node2 monitor=30000 on 18builder * Resource action: nfs_mount monitor=10000 on 18node2 Revised cluster status: Online: [ 18builder ] Containers: [ 18node2:container2 ] fence_false (stonith:fence_false): Stopped container2 (ocf::pacemaker:Dummy): Started 18builder webserver (ocf::pacemaker:Dummy): Stopped nfs_mount (ocf::pacemaker:Dummy): Started 18node2 Resource Group: mygroup vg_tags (ocf::heartbeat:LVM): Stopped ( disabled ) vg_tags_dup (ocf::heartbeat:LVM): Stopped diff --git a/pengine/test10/whitebox-fail1.dot b/pengine/test10/whitebox-fail1.dot index 0123a588b0..3cf4f7689d 100644 --- a/pengine/test10/whitebox-fail1.dot +++ b/pengine/test10/whitebox-fail1.dot @@ -1,56 +1,61 @@ digraph "g" { +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] +"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" -> "M-clone_start_0" [ style = bold] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "M_start_0 lxc1" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] "container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "lxc1_monitor_30000 18node2" [ style = bold] "lxc1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "B_start_0 lxc1" [ style = bold] "stonith_complete" -> "M_start_0 lxc1" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 18node2" [ style = bold] "stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-fail1.exp b/pengine/test10/whitebox-fail1.exp index 4cb6136b0e..ab739b5fd9 100644 --- a/pengine/test10/whitebox-fail1.exp +++ b/pengine/test10/whitebox-fail1.exp @@ -1,287 +1,320 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + + + + - + - + - + - + - + - + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary index 1271d4f9a4..d1f34804e4 100644 --- a/pengine/test10/whitebox-fail1.summary +++ b/pengine/test10/whitebox-fail1.summary @@ -1,56 +1,59 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): FAILED 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): FAILED lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Recover container1 ( 18node2 ) * Recover M:4 ( lxc1 ) * Recover B ( lxc1 ) * Restart lxc1 ( 18node2 ) due to required container1 start Executing cluster transition: + * Resource action: A monitor on lxc2 + * Resource action: B monitor on lxc2 + * Resource action: D monitor on lxc2 * Resource action: lxc1 stop on 18node2 * Resource action: container1 stop on 18node2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: container1 start on 18node2 * Pseudo action: M-clone_stop_0 * Pseudo action: B_stop_0 * Resource action: lxc1 start on 18node2 * Resource action: lxc1 monitor=30000 on 18node2 * Pseudo action: M_stop_0 * Pseudo action: M-clone_stopped_0 * Pseudo action: M-clone_start_0 * Resource action: B start on lxc1 * Pseudo action: all_stopped * Resource action: M start on lxc1 * Pseudo action: M-clone_running_0 * Resource action: B monitor=10000 on lxc1 * Resource action: M monitor=10000 on lxc1 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-fail2.dot b/pengine/test10/whitebox-fail2.dot index 0123a588b0..3cf4f7689d 100644 --- a/pengine/test10/whitebox-fail2.dot +++ b/pengine/test10/whitebox-fail2.dot @@ -1,56 +1,61 @@ digraph "g" { +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] +"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" -> "M-clone_start_0" [ style = bold] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "M_start_0 lxc1" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] "container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "lxc1_monitor_30000 18node2" [ style = bold] "lxc1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "B_start_0 lxc1" [ style = bold] "stonith_complete" -> "M_start_0 lxc1" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 18node2" [ style = bold] "stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-fail2.exp b/pengine/test10/whitebox-fail2.exp index 4cb6136b0e..ab739b5fd9 100644 --- a/pengine/test10/whitebox-fail2.exp +++ b/pengine/test10/whitebox-fail2.exp @@ -1,287 +1,320 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + + + + - + - + - + - + - + - + + + + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary index 5fd1ebd764..ebf6c515e3 100644 --- a/pengine/test10/whitebox-fail2.summary +++ b/pengine/test10/whitebox-fail2.summary @@ -1,56 +1,59 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): FAILED 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): FAILED lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Recover container1 ( 18node2 ) * Recover M:4 ( lxc1 ) * Recover B ( lxc1 ) * Recover lxc1 ( 18node2 ) Executing cluster transition: + * Resource action: A monitor on lxc2 + * Resource action: B monitor on lxc2 + * Resource action: D monitor on lxc2 * Resource action: lxc1 stop on 18node2 * Resource action: container1 stop on 18node2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: container1 start on 18node2 * Pseudo action: M-clone_stop_0 * Pseudo action: B_stop_0 * Resource action: lxc1 start on 18node2 * Resource action: lxc1 monitor=30000 on 18node2 * Pseudo action: M_stop_0 * Pseudo action: M-clone_stopped_0 * Pseudo action: M-clone_start_0 * Resource action: B start on lxc1 * Pseudo action: all_stopped * Resource action: M start on lxc1 * Pseudo action: M-clone_running_0 * Resource action: B monitor=10000 on lxc1 * Resource action: M monitor=10000 on lxc1 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-fail3.dot b/pengine/test10/whitebox-fail3.dot index 9814f660f1..6f608d101e 100644 --- a/pengine/test10/whitebox-fail3.dot +++ b/pengine/test10/whitebox-fail3.dot @@ -1,38 +1,41 @@ digraph "g" { "18builder_monitor_0 dvossel-laptop2" -> "18builder_start_0 dvossel-laptop2" [ style = bold] +"18builder_monitor_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] +"18builder_monitor_0 dvossel-laptop2" -> "W-master_start_0" [ style = bold] +"18builder_monitor_0 dvossel-laptop2" -> "X-master_start_0" [ style = bold] "18builder_monitor_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] "18builder_monitor_30000 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] "18builder_start_0 dvossel-laptop2" -> "18builder_monitor_30000 dvossel-laptop2" [ style = bold] "18builder_start_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] "18builder_start_0 dvossel-laptop2" -> "W:1_monitor_10000 18builder" [ style = bold] "18builder_start_0 dvossel-laptop2" -> "W:1_start_0 18builder" [ style = bold] "18builder_start_0 dvossel-laptop2" -> "X:1_monitor_10000 18builder" [ style = bold] "18builder_start_0 dvossel-laptop2" -> "X:1_start_0 18builder" [ style = bold] "18builder_start_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] "FAKE_start_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE_stop_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] "FAKE_stop_0 dvossel-laptop2" -> "all_stopped" [ style = bold] "FAKE_stop_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] "W-master_running_0" [ style=bold color="green" fontcolor="orange"] "W-master_start_0" -> "W-master_running_0" [ style = bold] "W-master_start_0" -> "W:1_start_0 18builder" [ style = bold] "W-master_start_0" [ style=bold color="green" fontcolor="orange"] "W:1_monitor_10000 18builder" [ style=bold color="green" fontcolor="black"] "W:1_start_0 18builder" -> "W-master_running_0" [ style = bold] "W:1_start_0 18builder" -> "W:1_monitor_10000 18builder" [ style = bold] "W:1_start_0 18builder" [ style=bold color="green" fontcolor="black"] "X-master_running_0" [ style=bold color="green" fontcolor="orange"] "X-master_start_0" -> "X-master_running_0" [ style = bold] "X-master_start_0" -> "X:1_start_0 18builder" [ style = bold] "X-master_start_0" [ style=bold color="green" fontcolor="orange"] "X:1_monitor_10000 18builder" [ style=bold color="green" fontcolor="black"] "X:1_start_0 18builder" -> "X-master_running_0" [ style = bold] "X:1_start_0 18builder" -> "X:1_monitor_10000 18builder" [ style = bold] "X:1_start_0 18builder" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "vm_start_0 dvossel-laptop2" -> "18builder_start_0 dvossel-laptop2" [ style = bold] "vm_start_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] "vm_start_0 dvossel-laptop2" -> "W:1_start_0 18builder" [ style = bold] "vm_start_0 dvossel-laptop2" -> "X:1_start_0 18builder" [ style = bold] "vm_start_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-fail3.exp b/pengine/test10/whitebox-fail3.exp index ec28f12fb8..cddd7dda23 100644 --- a/pengine/test10/whitebox-fail3.exp +++ b/pengine/test10/whitebox-fail3.exp @@ -1,206 +1,217 @@ + + + - + + + + + - + + + + + diff --git a/pengine/test10/whitebox-fail3.summary b/pengine/test10/whitebox-fail3.summary index eded0999e0..374fb69103 100644 --- a/pengine/test10/whitebox-fail3.summary +++ b/pengine/test10/whitebox-fail3.summary @@ -1,54 +1,54 @@ Current cluster status: Online: [ dvossel-laptop2 ] vm (ocf::heartbeat:VirtualDomain): Stopped vm2 (ocf::heartbeat:VirtualDomain): Stopped FAKE (ocf::pacemaker:Dummy): Started dvossel-laptop2 Master/Slave Set: W-master [W] Masters: [ dvossel-laptop2 ] Stopped: [ 18builder 18node1 ] Master/Slave Set: X-master [X] Masters: [ dvossel-laptop2 ] Stopped: [ 18builder 18node1 ] Transition Summary: * Start vm (dvossel-laptop2) * Move FAKE ( dvossel-laptop2 -> 18builder ) * Start W:1 (18builder) * Start X:1 (18builder) * Start 18builder (dvossel-laptop2) Executing cluster transition: * Resource action: vm start on dvossel-laptop2 * Resource action: FAKE stop on dvossel-laptop2 - * Pseudo action: W-master_start_0 - * Pseudo action: X-master_start_0 * Resource action: 18builder monitor on dvossel-laptop2 * Pseudo action: all_stopped + * Pseudo action: W-master_start_0 + * Pseudo action: X-master_start_0 * Resource action: 18builder start on dvossel-laptop2 * Resource action: FAKE start on 18builder * Resource action: W start on 18builder * Pseudo action: W-master_running_0 * Resource action: X start on 18builder * Pseudo action: X-master_running_0 * Resource action: 18builder monitor=30000 on dvossel-laptop2 * Resource action: W monitor=10000 on 18builder * Resource action: X monitor=10000 on 18builder Revised cluster status: Online: [ dvossel-laptop2 ] Containers: [ 18builder:vm ] vm (ocf::heartbeat:VirtualDomain): Started dvossel-laptop2 vm2 (ocf::heartbeat:VirtualDomain): Stopped FAKE (ocf::pacemaker:Dummy): Started 18builder Master/Slave Set: W-master [W] Masters: [ dvossel-laptop2 ] Slaves: [ 18builder ] Stopped: [ 18node1 ] Master/Slave Set: X-master [X] Masters: [ dvossel-laptop2 ] Slaves: [ 18builder ] Stopped: [ 18node1 ] diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot index 8ef42fd9fb..029e9f17a3 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.dot +++ b/pengine/test10/whitebox-imply-stop-on-fence.dot @@ -1,100 +1,119 @@ digraph "g" { "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-01_kiff-01_start_0 kiff-02" -> "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] "R-lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "all_stopped" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] "clvmd-clone_stop_0" -> "clvmd_stop_0 kiff-01" [ style = bold] "clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"clvmd_monitor_0 lxc-01_kiff-02" -> "clvmd_stop_0 kiff-01" [ style = bold] +"clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"clvmd_monitor_0 lxc-02_kiff-02" -> "clvmd_stop_0 kiff-01" [ style = bold] +"clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "clvmd_stop_0 kiff-01" -> "all_stopped" [ style = bold] "clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] "clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] "clvmd_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] "dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] "dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"dlm_monitor_0 lxc-01_kiff-02" -> "dlm_stop_0 kiff-01" [ style = bold] +"dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"dlm_monitor_0 lxc-02_kiff-02" -> "dlm_stop_0 kiff-01" [ style = bold] +"dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "dlm_stop_0 kiff-01" -> "all_stopped" [ style = bold] "dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] "dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "fence-kiff-02_monitor_60000 kiff-02" [ style=bold color="green" fontcolor="black"] "fence-kiff-02_start_0 kiff-02" -> "fence-kiff-02_monitor_60000 kiff-02" [ style = bold] "fence-kiff-02_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "fence-kiff-02_stop_0 kiff-01" -> "all_stopped" [ style = bold] "fence-kiff-02_stop_0 kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] "fence-kiff-02_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "lxc-01_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_monitor_30000 kiff-02" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "lxc-02_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_monitor_30000 kiff-02" [ style = bold] "lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +"lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] "shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] "shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] "shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +"shared0_monitor_0 lxc-01_kiff-02" -> "shared0_stop_0 kiff-01" [ style = bold] +"shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"shared0_monitor_0 lxc-02_kiff-02" -> "shared0_stop_0 kiff-01" [ style = bold] +"shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "shared0_stop_0 kiff-01" -> "all_stopped" [ style = bold] "shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] "shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] "shared0_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "clvmd-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "dlm-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc-01_kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] "stonith 'reboot' lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc-02_kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc-02_kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] +"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] +"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] +"vm-fs_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "vm-fs_monitor_20000 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] "vm-fs_start_0 lxc-01_kiff-01" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] "vm-fs_start_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] "vm-fs_stop_0 lxc-01_kiff-01" -> "all_stopped" [ style = bold] "vm-fs_stop_0 lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "vm-fs_stop_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp index d1e22eeed6..b5700c095b 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.exp +++ b/pengine/test10/whitebox-imply-stop-on-fence.exp @@ -1,509 +1,614 @@ - + - + - + - + - + - + - + - + + + + + + + - + + + + + + + + + + - + + + + + + + + + + - + - + - + - + - + - + - + - + + + + + + + - + - + + + + + + + + + + - + + + + + + + + + + - + - + - + - + - + - + - + - + + + + + + + - + - + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + + + + + + + - + + + + + + + + + + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary index 87d7b4f320..d272b25d21 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.summary +++ b/pengine/test10/whitebox-imply-stop-on-fence.summary @@ -1,96 +1,104 @@ Current cluster status: Node kiff-01 (1): UNCLEAN (offline) Online: [ kiff-02 ] Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 fence-kiff-02 (stonith:fence_ipmilan): Started kiff-01 (UNCLEAN) Clone Set: dlm-clone [dlm] dlm (ocf::pacemaker:controld): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: clvmd-clone [clvmd] clvmd (ocf::heartbeat:clvm): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: shared0-clone [shared0] shared0 (ocf::heartbeat:Filesystem): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): FAILED kiff-01 (UNCLEAN) R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 Transition Summary: * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' * Fence (reboot) kiff-01 'peer is no longer part of the cluster' * Move fence-kiff-02 ( kiff-01 -> kiff-02 ) * Stop dlm:0 (kiff-01) due to node availability * Stop clvmd:0 (kiff-01) due to node availability * Stop shared0:0 (kiff-01) due to node availability * Recover R-lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) * Move R-lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) * Recover vm-fs ( lxc-01_kiff-01 ) * Move lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) * Move lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) Executing cluster transition: * Pseudo action: fence-kiff-02_stop_0 + * Resource action: dlm monitor on lxc-02_kiff-02 + * Resource action: dlm monitor on lxc-01_kiff-02 + * Resource action: clvmd monitor on lxc-02_kiff-02 + * Resource action: clvmd monitor on lxc-01_kiff-02 + * Resource action: shared0 monitor on lxc-02_kiff-02 + * Resource action: shared0 monitor on lxc-01_kiff-02 + * Resource action: vm-fs monitor on lxc-02_kiff-02 + * Resource action: vm-fs monitor on lxc-01_kiff-02 * Fencing kiff-01 (reboot) * Pseudo action: lxc-01_kiff-01_stop_0 * Pseudo action: lxc-02_kiff-01_stop_0 * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 * Pseudo action: stonith_complete * Pseudo action: R-lxc-01_kiff-01_stop_0 * Pseudo action: R-lxc-02_kiff-01_stop_0 * Pseudo action: vm-fs_stop_0 * Pseudo action: shared0-clone_stop_0 * Resource action: R-lxc-01_kiff-01 start on kiff-02 * Resource action: R-lxc-02_kiff-01 start on kiff-02 * Resource action: lxc-01_kiff-01 start on kiff-02 * Resource action: lxc-02_kiff-01 start on kiff-02 * Pseudo action: shared0_stop_0 * Pseudo action: shared0-clone_stopped_0 * Resource action: R-lxc-01_kiff-01 monitor=10000 on kiff-02 * Resource action: R-lxc-02_kiff-01 monitor=10000 on kiff-02 * Resource action: vm-fs start on lxc-01_kiff-01 * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 * Pseudo action: clvmd-clone_stop_0 * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 * Pseudo action: clvmd_stop_0 * Pseudo action: clvmd-clone_stopped_0 * Pseudo action: dlm-clone_stop_0 * Pseudo action: dlm_stop_0 * Pseudo action: dlm-clone_stopped_0 * Pseudo action: all_stopped * Resource action: fence-kiff-02 start on kiff-02 * Resource action: fence-kiff-02 monitor=60000 on kiff-02 Revised cluster status: Online: [ kiff-02 ] OFFLINE: [ kiff-01 ] Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-01:R-lxc-02_kiff-01 lxc-02_kiff-02:R-lxc-02_kiff-02 ] fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 fence-kiff-02 (stonith:fence_ipmilan): Started kiff-02 Clone Set: dlm-clone [dlm] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: clvmd-clone [clvmd] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: shared0-clone [shared0] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 vm-fs (ocf::heartbeat:Filesystem): Started lxc-01_kiff-01 diff --git a/pengine/test10/whitebox-migrate1.dot b/pengine/test10/whitebox-migrate1.dot index e54df2c1d1..53d37e0de4 100644 --- a/pengine/test10/whitebox-migrate1.dot +++ b/pengine/test10/whitebox-migrate1.dot @@ -1,46 +1,47 @@ digraph "g" { "FAKE3_monitor_10000 rhel7-node2" [ style=bold color="green" fontcolor="black"] "FAKE3_start_0 rhel7-node2" -> "FAKE3_monitor_10000 rhel7-node2" [ style = bold] "FAKE3_start_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "FAKE3_stop_0 rhel7-node3" -> "FAKE3_start_0 rhel7-node2" [ style = bold] "FAKE3_stop_0 rhel7-node3" -> "all_stopped" [ style = bold] "FAKE3_stop_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "remote-rsc_migrate_from_0 rhel7-node3" -> "remote-rsc_start_0 rhel7-node3" [ style = bold] "remote-rsc_migrate_from_0 rhel7-node3" -> "remote-rsc_stop_0 rhel7-node2" [ style = bold] "remote-rsc_migrate_from_0 rhel7-node3" -> "rhel7-node1_migrate_to_0 rhel7-node2" [ style = bold] "remote-rsc_migrate_from_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] "remote-rsc_migrate_to_0 rhel7-node2" -> "remote-rsc_migrate_from_0 rhel7-node3" [ style = bold] "remote-rsc_migrate_to_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "remote-rsc_monitor_10000 rhel7-node3" [ style=bold color="green" fontcolor="black"] "remote-rsc_start_0 rhel7-node3" -> "remote-rsc_monitor_10000 rhel7-node3" [ style = bold] "remote-rsc_start_0 rhel7-node3" -> "rhel7-node1_migrate_to_0 rhel7-node2" [ style = dashed] "remote-rsc_start_0 rhel7-node3" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] "remote-rsc_start_0 rhel7-node3" [ style=bold color="green" fontcolor="orange"] "remote-rsc_stop_0 rhel7-node2" -> "all_stopped" [ style = bold] "remote-rsc_stop_0 rhel7-node2" -> "remote-rsc_start_0 rhel7-node3" [ style = bold] "remote-rsc_stop_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "rhel7-node1_migrate_from_0 rhel7-node3" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] "rhel7-node1_migrate_from_0 rhel7-node3" -> "rhel7-node1_stop_0 rhel7-node2" [ style = bold] "rhel7-node1_migrate_from_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] "rhel7-node1_migrate_to_0 rhel7-node2" -> "rhel7-node1_migrate_from_0 rhel7-node3" [ style = bold] "rhel7-node1_migrate_to_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "rhel7-node1_monitor_0 rhel7-node3" -> "rhel7-node1_migrate_to_0 rhel7-node2" [ style = bold] "rhel7-node1_monitor_0 rhel7-node3" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] "rhel7-node1_monitor_0 rhel7-node3" -> "rhel7-node1_stop_0 rhel7-node2" [ style = bold] "rhel7-node1_monitor_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] "rhel7-node1_monitor_30000 rhel7-node3" [ style=bold color="green" fontcolor="black"] "rhel7-node1_start_0 rhel7-node3" -> "rhel7-node1_monitor_30000 rhel7-node3" [ style = bold] "rhel7-node1_start_0 rhel7-node3" [ style=bold color="green" fontcolor="orange"] +"rhel7-node1_stop_0 rhel7-node2" -> "FAKE3_start_0 rhel7-node2" [ style = dashed] "rhel7-node1_stop_0 rhel7-node2" -> "all_stopped" [ style = bold] "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_migrate_to_0 rhel7-node2" [ style = dashed] "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_stop_0 rhel7-node2" [ style = bold] "rhel7-node1_stop_0 rhel7-node2" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] "rhel7-node1_stop_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "shooter1_monitor_60000 rhel7-node2" [ style=bold color="green" fontcolor="black"] "shooter1_start_0 rhel7-node2" -> "shooter1_monitor_60000 rhel7-node2" [ style = bold] "shooter1_start_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] "shooter1_stop_0 rhel7-node3" -> "all_stopped" [ style = bold] "shooter1_stop_0 rhel7-node3" -> "shooter1_start_0 rhel7-node2" [ style = bold] "shooter1_stop_0 rhel7-node3" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-move.dot b/pengine/test10/whitebox-move.dot index f47c95b97f..ed06b7c61a 100644 --- a/pengine/test10/whitebox-move.dot +++ b/pengine/test10/whitebox-move.dot @@ -1,46 +1,49 @@ digraph "g" { +"A_monitor_0 lxc2" -> "A_start_0 lxc1" [ style = bold] +"A_monitor_0 lxc2" -> "A_stop_0 lxc1" [ style = bold] +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "A_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "A_start_0 lxc1" -> "A_monitor_10000 lxc1" [ style = bold] "A_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "A_stop_0 lxc1" -> "A_start_0 lxc1" [ style = bold] "A_stop_0 lxc1" -> "all_stopped" [ style = bold] "A_stop_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] "A_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" -> "M-clone_start_0" [ style = bold] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "M_start_0 lxc1" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node2" -> "A_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node1" -> "all_stopped" [ style = bold] "container1_stop_0 18node1" -> "container1_start_0 18node2" [ style = bold] "container1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node2" -> "A_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "A_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "lxc1_monitor_30000 18node2" [ style = bold] "lxc1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node1" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node1" -> "container1_stop_0 18node1" [ style = bold] "lxc1_stop_0 18node1" -> "lxc1_start_0 18node2" [ style = bold] "lxc1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-move.exp b/pengine/test10/whitebox-move.exp index 446189088a..6f0027525c 100644 --- a/pengine/test10/whitebox-move.exp +++ b/pengine/test10/whitebox-move.exp @@ -1,243 +1,259 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-move.summary b/pengine/test10/whitebox-move.summary index 5e27a67772..23bc5de174 100644 --- a/pengine/test10/whitebox-move.summary +++ b/pengine/test10/whitebox-move.summary @@ -1,47 +1,48 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node1 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started lxc1 Transition Summary: * Move container1 ( 18node1 -> 18node2 ) * Restart M:3 ( lxc1 ) due to required container1 start * Restart A ( lxc1 ) due to required container1 start * Move lxc1 ( 18node1 -> 18node2 ) Executing cluster transition: * Pseudo action: M-clone_stop_0 - * Resource action: A stop on lxc1 + * Resource action: A monitor on lxc2 * Resource action: M stop on lxc1 * Pseudo action: M-clone_stopped_0 * Pseudo action: M-clone_start_0 + * Resource action: A stop on lxc1 * Resource action: lxc1 stop on 18node1 * Resource action: container1 stop on 18node1 * Pseudo action: all_stopped * Resource action: container1 start on 18node2 * Resource action: lxc1 start on 18node2 * Resource action: M start on lxc1 * Resource action: M monitor=10000 on lxc1 * Pseudo action: M-clone_running_0 * Resource action: A start on lxc1 * Resource action: A monitor=10000 on lxc1 * Resource action: lxc1 monitor=30000 on 18node2 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started lxc1 diff --git a/pengine/test10/whitebox-ms-ordering-move.dot b/pengine/test10/whitebox-ms-ordering-move.dot index 43c1d4f6a2..e6b5406904 100644 --- a/pengine/test10/whitebox-ms-ordering-move.dot +++ b/pengine/test10/whitebox-ms-ordering-move.dot @@ -1,71 +1,83 @@ digraph "g" { "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] "container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"] +"lsb-dummy_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] "lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] "lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] "lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-2" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-3" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-3" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-4" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-4" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-5" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-5" -> "lxc1_stop_0 rhel7-1" [ style = bold] "lxc1_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc1_monitor_30000 rhel7-2" [ style = bold] "lxc1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] "lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] +"migrator_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"petulant_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"ping-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"r192.168.122.207_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"r192.168.122.208_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"rsc_rhel7-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"rsc_rhel7-2_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"rsc_rhel7-3_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"rsc_rhel7-4_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"rsc_rhel7-5_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"stateful-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-ms-ordering-move.exp b/pengine/test10/whitebox-ms-ordering-move.exp index dc5e4738b5..a8ffa641e2 100644 --- a/pengine/test10/whitebox-ms-ordering-move.exp +++ b/pengine/test10/whitebox-ms-ordering-move.exp @@ -1,376 +1,484 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-ms-ordering-move.summary b/pengine/test10/whitebox-ms-ordering-move.summary index 80156b0fda..af86d7472a 100644 --- a/pengine/test10/whitebox-ms-ordering-move.summary +++ b/pengine/test10/whitebox-ms-ordering-move.summary @@ -1,94 +1,106 @@ Current cluster status: Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Containers: [ lxc1:container1 lxc2:container2 ] Fencing (stonith:fence_xvm): Started rhel7-3 FencingPass (stonith:fence_dummy): Started rhel7-4 FencingFail (stonith:fence_dummy): Started rhel7-5 rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-3 rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 migrator (ocf::pacemaker:Dummy): Started rhel7-4 Clone Set: Connectivity [ping-1] Started: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Stopped: [ lxc1 lxc2 ] Master/Slave Set: master-1 [stateful-1] Masters: [ rhel7-3 ] Slaves: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] Resource Group: group-1 r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-3 petulant (service:DummySD): Started rhel7-3 r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-3 lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-3 container1 (ocf::heartbeat:VirtualDomain): Started rhel7-1 container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1 Master/Slave Set: lxc-ms-master [lxc-ms] Masters: [ lxc1 ] Slaves: [ lxc2 ] Transition Summary: * Move container1 ( rhel7-1 -> rhel7-2 ) * Restart lxc-ms:0 (Master lxc1) due to required container1 start * Move lxc1 ( rhel7-1 -> rhel7-2 ) Executing cluster transition: + * Resource action: rsc_rhel7-1 monitor on lxc2 + * Resource action: rsc_rhel7-2 monitor on lxc2 + * Resource action: rsc_rhel7-3 monitor on lxc2 + * Resource action: rsc_rhel7-4 monitor on lxc2 + * Resource action: rsc_rhel7-5 monitor on lxc2 + * Resource action: migrator monitor on lxc2 + * Resource action: ping-1 monitor on lxc2 + * Resource action: stateful-1 monitor on lxc2 + * Resource action: r192.168.122.207 monitor on lxc2 + * Resource action: petulant monitor on lxc2 + * Resource action: r192.168.122.208 monitor on lxc2 + * Resource action: lsb-dummy monitor on lxc2 * Pseudo action: lxc-ms-master_demote_0 * Resource action: lxc1 monitor on rhel7-5 * Resource action: lxc1 monitor on rhel7-4 * Resource action: lxc1 monitor on rhel7-3 * Resource action: lxc1 monitor on rhel7-2 * Resource action: lxc2 monitor on rhel7-5 * Resource action: lxc2 monitor on rhel7-4 * Resource action: lxc2 monitor on rhel7-3 * Resource action: lxc2 monitor on rhel7-2 * Resource action: lxc-ms demote on lxc1 * Pseudo action: lxc-ms-master_demoted_0 * Pseudo action: lxc-ms-master_stop_0 * Resource action: lxc-ms stop on lxc1 * Pseudo action: lxc-ms-master_stopped_0 * Pseudo action: lxc-ms-master_start_0 * Resource action: lxc1 stop on rhel7-1 * Resource action: container1 stop on rhel7-1 * Pseudo action: all_stopped * Resource action: container1 start on rhel7-2 * Resource action: lxc1 start on rhel7-2 * Resource action: lxc-ms start on lxc1 * Pseudo action: lxc-ms-master_running_0 * Resource action: lxc1 monitor=30000 on rhel7-2 * Pseudo action: lxc-ms-master_promote_0 * Resource action: lxc-ms promote on lxc1 * Pseudo action: lxc-ms-master_promoted_0 Revised cluster status: Online: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Containers: [ lxc1:container1 lxc2:container2 ] Fencing (stonith:fence_xvm): Started rhel7-3 FencingPass (stonith:fence_dummy): Started rhel7-4 FencingFail (stonith:fence_dummy): Started rhel7-5 rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-3 rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 migrator (ocf::pacemaker:Dummy): Started rhel7-4 Clone Set: Connectivity [ping-1] Started: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Stopped: [ lxc1 lxc2 ] Master/Slave Set: master-1 [stateful-1] Masters: [ rhel7-3 ] Slaves: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] Resource Group: group-1 r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-3 petulant (service:DummySD): Started rhel7-3 r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-3 lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-3 container1 (ocf::heartbeat:VirtualDomain): Started rhel7-2 container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1 Master/Slave Set: lxc-ms-master [lxc-ms] Masters: [ lxc1 ] Slaves: [ lxc2 ] diff --git a/pengine/test10/whitebox-nested-group.dot b/pengine/test10/whitebox-nested-group.dot index 9e1abce729..e5749ecf5a 100644 --- a/pengine/test10/whitebox-nested-group.dot +++ b/pengine/test10/whitebox-nested-group.dot @@ -1,115 +1,136 @@ digraph "g" { "c7auto4_monitor_0 c7auto1" -> "c7auto4_start_0 c7auto1" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake1_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake3_start_0 c7auto2" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake4_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake_clone_start_0" [ style = bold] +"c7auto4_monitor_0 c7auto1" -> "fake_group_start_0" [ style = bold] "c7auto4_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "c7auto4_monitor_0 c7auto2" -> "c7auto4_start_0 c7auto1" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake1_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake2_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake3_start_0 c7auto2" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake4_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake5_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake_clone_start_0" [ style = bold] +"c7auto4_monitor_0 c7auto2" -> "fake_group_start_0" [ style = bold] "c7auto4_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "c7auto4_monitor_0 c7auto3" -> "c7auto4_start_0 c7auto1" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake1_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake2_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake3_start_0 c7auto2" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake4_start_0 c7auto3" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake5_start_0 c7auto4" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake_clone_start_0" [ style = bold] +"c7auto4_monitor_0 c7auto3" -> "fake_group_start_0" [ style = bold] "c7auto4_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "c7auto4_monitor_30000 c7auto1" [ style=bold color="green" fontcolor="black"] "c7auto4_start_0 c7auto1" -> "c7auto4_monitor_30000 c7auto1" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake2_monitor_10000 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake5_monitor_10000 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake:2_monitor_10000 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" -> "fake:2_start_0 c7auto4" [ style = bold] "c7auto4_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] "container_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] "container_start_0 c7auto1" -> "c7auto4_start_0 c7auto1" [ style = bold] "container_start_0 c7auto1" -> "container_monitor_10000 c7auto1" [ style = bold] "container_start_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] "container_start_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] "container_start_0 c7auto1" -> "fake:2_start_0 c7auto4" [ style = bold] "container_start_0 c7auto1" -> "fake_group_running_0" [ style = bold] "container_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake1_monitor_0 c7auto1" -> "fake1_start_0 c7auto3" [ style = bold] "fake1_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake1_monitor_0 c7auto2" -> "fake1_start_0 c7auto3" [ style = bold] "fake1_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake1_monitor_0 c7auto3" -> "fake1_start_0 c7auto3" [ style = bold] "fake1_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake1_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"] "fake1_start_0 c7auto3" -> "fake1_monitor_10000 c7auto3" [ style = bold] "fake1_start_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake2_monitor_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] "fake2_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake2_monitor_0 c7auto2" -> "fake2_start_0 c7auto4" [ style = bold] "fake2_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake2_monitor_0 c7auto3" -> "fake2_start_0 c7auto4" [ style = bold] "fake2_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake2_monitor_10000 c7auto4" [ style=bold color="green" fontcolor="black"] "fake2_start_0 c7auto4" -> "fake2_monitor_10000 c7auto4" [ style = bold] "fake2_start_0 c7auto4" [ style=bold color="green" fontcolor="black"] "fake3_monitor_0 c7auto1" -> "fake3_start_0 c7auto2" [ style = bold] "fake3_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake3_monitor_0 c7auto2" -> "fake3_start_0 c7auto2" [ style = bold] "fake3_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake3_monitor_0 c7auto3" -> "fake3_start_0 c7auto2" [ style = bold] "fake3_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake3_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"] "fake3_start_0 c7auto2" -> "fake3_monitor_10000 c7auto2" [ style = bold] "fake3_start_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake4_monitor_0 c7auto1" -> "fake4_start_0 c7auto3" [ style = bold] "fake4_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake4_monitor_0 c7auto2" -> "fake4_start_0 c7auto3" [ style = bold] "fake4_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake4_monitor_0 c7auto3" -> "fake4_start_0 c7auto3" [ style = bold] "fake4_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake4_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"] "fake4_start_0 c7auto3" -> "fake4_monitor_10000 c7auto3" [ style = bold] "fake4_start_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake5_monitor_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] "fake5_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake5_monitor_0 c7auto2" -> "fake5_start_0 c7auto4" [ style = bold] "fake5_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake5_monitor_0 c7auto3" -> "fake5_start_0 c7auto4" [ style = bold] "fake5_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake5_monitor_10000 c7auto4" [ style=bold color="green" fontcolor="black"] "fake5_start_0 c7auto4" -> "fake5_monitor_10000 c7auto4" [ style = bold] "fake5_start_0 c7auto4" [ style=bold color="green" fontcolor="black"] "fake:0_monitor_0 c7auto2" -> "fake_clone_start_0" [ style = bold] "fake:0_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake:0_monitor_10000 c7auto2" [ style=bold color="green" fontcolor="black"] "fake:0_start_0 c7auto2" -> "fake:0_monitor_10000 c7auto2" [ style = bold] "fake:0_start_0 c7auto2" -> "fake_clone_running_0" [ style = bold] "fake:0_start_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake:1_monitor_0 c7auto3" -> "fake_clone_start_0" [ style = bold] "fake:1_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake:1_monitor_10000 c7auto3" [ style=bold color="green" fontcolor="black"] "fake:1_start_0 c7auto3" -> "fake:1_monitor_10000 c7auto3" [ style = bold] "fake:1_start_0 c7auto3" -> "fake_clone_running_0" [ style = bold] "fake:1_start_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake:2_monitor_10000 c7auto4" [ style=bold color="green" fontcolor="black"] "fake:2_start_0 c7auto4" -> "fake:2_monitor_10000 c7auto4" [ style = bold] "fake:2_start_0 c7auto4" -> "fake_clone_running_0" [ style = bold] "fake:2_start_0 c7auto4" [ style=bold color="green" fontcolor="black"] "fake:3_monitor_0 c7auto1" -> "fake_clone_start_0" [ style = bold] "fake:3_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake:3_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] "fake:3_start_0 c7auto1" -> "fake:3_monitor_10000 c7auto1" [ style = bold] "fake:3_start_0 c7auto1" -> "fake_clone_running_0" [ style = bold] "fake:3_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake_clone_running_0" [ style=bold color="green" fontcolor="orange"] "fake_clone_start_0" -> "fake:0_start_0 c7auto2" [ style = bold] "fake_clone_start_0" -> "fake:1_start_0 c7auto3" [ style = bold] "fake_clone_start_0" -> "fake:2_start_0 c7auto4" [ style = bold] "fake_clone_start_0" -> "fake:3_start_0 c7auto1" [ style = bold] "fake_clone_start_0" -> "fake_clone_running_0" [ style = bold] "fake_clone_start_0" [ style=bold color="green" fontcolor="orange"] "fake_fs_monitor_0 c7auto1" -> "fake_fs_start_0 c7auto1" [ style = bold] "fake_fs_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake_fs_monitor_0 c7auto2" -> "fake_fs_start_0 c7auto1" [ style = bold] "fake_fs_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] "fake_fs_monitor_0 c7auto3" -> "fake_fs_start_0 c7auto1" [ style = bold] "fake_fs_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] "fake_fs_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"] "fake_fs_start_0 c7auto1" -> "container_start_0 c7auto1" [ style = bold] "fake_fs_start_0 c7auto1" -> "fake_fs_monitor_10000 c7auto1" [ style = bold] "fake_fs_start_0 c7auto1" -> "fake_group_running_0" [ style = bold] "fake_fs_start_0 c7auto1" [ style=bold color="green" fontcolor="black"] "fake_group_running_0" [ style=bold color="green" fontcolor="orange"] "fake_group_start_0" -> "container_start_0 c7auto1" [ style = bold] "fake_group_start_0" -> "fake_fs_start_0 c7auto1" [ style = bold] "fake_group_start_0" -> "fake_group_running_0" [ style = bold] "fake_group_start_0" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-nested-group.exp b/pengine/test10/whitebox-nested-group.exp index 388c4d467d..5625c92561 100644 --- a/pengine/test10/whitebox-nested-group.exp +++ b/pengine/test10/whitebox-nested-group.exp @@ -1,676 +1,740 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + diff --git a/pengine/test10/whitebox-nested-group.summary b/pengine/test10/whitebox-nested-group.summary index ca9c47f13e..b03b357c96 100644 --- a/pengine/test10/whitebox-nested-group.summary +++ b/pengine/test10/whitebox-nested-group.summary @@ -1,100 +1,100 @@ Current cluster status: Online: [ c7auto1 c7auto2 c7auto3 ] shooter (stonith:fence_phd_kvm): Started c7auto2 fake1 (ocf::heartbeat:Dummy): Stopped fake2 (ocf::heartbeat:Dummy): Stopped fake3 (ocf::heartbeat:Dummy): Stopped fake4 (ocf::heartbeat:Dummy): Stopped fake5 (ocf::heartbeat:Dummy): Stopped Clone Set: fake_clone [fake] Stopped: [ c7auto1 c7auto2 c7auto3 c7auto4 ] Resource Group: fake_group fake_fs (ocf::heartbeat:Dummy): Stopped container (ocf::heartbeat:Dummy): Stopped Transition Summary: * Start fake1 (c7auto3) * Start fake2 (c7auto4) * Start fake3 (c7auto2) * Start fake4 (c7auto3) * Start fake5 (c7auto4) * Start fake:0 (c7auto2) * Start fake:1 (c7auto3) * Start fake:2 (c7auto4) * Start fake:3 (c7auto1) * Start fake_fs (c7auto1) * Start container (c7auto1) * Start c7auto4 (c7auto1) Executing cluster transition: * Resource action: fake1 monitor on c7auto3 * Resource action: fake1 monitor on c7auto2 * Resource action: fake1 monitor on c7auto1 * Resource action: fake2 monitor on c7auto3 * Resource action: fake2 monitor on c7auto2 * Resource action: fake2 monitor on c7auto1 * Resource action: fake3 monitor on c7auto3 * Resource action: fake3 monitor on c7auto2 * Resource action: fake3 monitor on c7auto1 * Resource action: fake4 monitor on c7auto3 * Resource action: fake4 monitor on c7auto2 * Resource action: fake4 monitor on c7auto1 * Resource action: fake5 monitor on c7auto3 * Resource action: fake5 monitor on c7auto2 * Resource action: fake5 monitor on c7auto1 * Resource action: fake:0 monitor on c7auto2 * Resource action: fake:1 monitor on c7auto3 * Resource action: fake:3 monitor on c7auto1 - * Pseudo action: fake_clone_start_0 - * Pseudo action: fake_group_start_0 * Resource action: fake_fs monitor on c7auto3 * Resource action: fake_fs monitor on c7auto2 * Resource action: fake_fs monitor on c7auto1 * Resource action: c7auto4 monitor on c7auto3 * Resource action: c7auto4 monitor on c7auto2 * Resource action: c7auto4 monitor on c7auto1 * Resource action: fake1 start on c7auto3 * Resource action: fake3 start on c7auto2 * Resource action: fake4 start on c7auto3 - * Resource action: fake:0 start on c7auto2 - * Resource action: fake:1 start on c7auto3 - * Resource action: fake:3 start on c7auto1 + * Pseudo action: fake_clone_start_0 + * Pseudo action: fake_group_start_0 * Resource action: fake_fs start on c7auto1 * Resource action: container start on c7auto1 * Resource action: c7auto4 start on c7auto1 * Resource action: fake1 monitor=10000 on c7auto3 * Resource action: fake2 start on c7auto4 * Resource action: fake3 monitor=10000 on c7auto2 * Resource action: fake4 monitor=10000 on c7auto3 * Resource action: fake5 start on c7auto4 - * Resource action: fake:0 monitor=10000 on c7auto2 - * Resource action: fake:1 monitor=10000 on c7auto3 + * Resource action: fake:0 start on c7auto2 + * Resource action: fake:1 start on c7auto3 * Resource action: fake:2 start on c7auto4 - * Resource action: fake:3 monitor=10000 on c7auto1 + * Resource action: fake:3 start on c7auto1 * Pseudo action: fake_clone_running_0 * Pseudo action: fake_group_running_0 * Resource action: fake_fs monitor=10000 on c7auto1 * Resource action: container monitor=10000 on c7auto1 * Resource action: c7auto4 monitor=30000 on c7auto1 * Resource action: fake2 monitor=10000 on c7auto4 * Resource action: fake5 monitor=10000 on c7auto4 + * Resource action: fake:0 monitor=10000 on c7auto2 + * Resource action: fake:1 monitor=10000 on c7auto3 * Resource action: fake:2 monitor=10000 on c7auto4 + * Resource action: fake:3 monitor=10000 on c7auto1 Revised cluster status: Online: [ c7auto1 c7auto2 c7auto3 ] Containers: [ c7auto4:container ] shooter (stonith:fence_phd_kvm): Started c7auto2 fake1 (ocf::heartbeat:Dummy): Started c7auto3 fake2 (ocf::heartbeat:Dummy): Started c7auto4 fake3 (ocf::heartbeat:Dummy): Started c7auto2 fake4 (ocf::heartbeat:Dummy): Started c7auto3 fake5 (ocf::heartbeat:Dummy): Started c7auto4 Clone Set: fake_clone [fake] Started: [ c7auto1 c7auto2 c7auto3 c7auto4 ] Resource Group: fake_group fake_fs (ocf::heartbeat:Dummy): Started c7auto1 container (ocf::heartbeat:Dummy): Started c7auto1 diff --git a/pengine/test10/whitebox-orphan-ms.dot b/pengine/test10/whitebox-orphan-ms.dot index 46b6cdaf6e..4e2e211a22 100644 --- a/pengine/test10/whitebox-orphan-ms.dot +++ b/pengine/test10/whitebox-orphan-ms.dot @@ -1,72 +1,90 @@ digraph "g" { "FencingFail_start_0 18node1" [ style=bold color="green" fontcolor="black"] "FencingFail_stop_0 18node3" -> "FencingFail_start_0 18node1" [ style = bold] "FencingFail_stop_0 18node3" -> "all_stopped" [ style = bold] "FencingFail_stop_0 18node3" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_delete_0 18node1" -> "container1_start_0 " [ style = dashed] "container1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "container1_delete_0 18node2" -> "container1_start_0 " [ style = dashed] "container1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_delete_0 18node3" -> "container1_start_0 " [ style = dashed] "container1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "container1_start_0 " [ style=dashed color="red" fontcolor="black"] "container1_stop_0 18node1" -> "all_stopped" [ style = bold] "container1_stop_0 18node1" -> "container1_delete_0 18node1" [ style = bold] "container1_stop_0 18node1" -> "container1_delete_0 18node2" [ style = bold] "container1_stop_0 18node1" -> "container1_delete_0 18node3" [ style = bold] "container1_stop_0 18node1" -> "container1_start_0 " [ style = dashed] "container1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] "container2_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "container2_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "container2_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "container2_stop_0 18node1" -> "all_stopped" [ style = bold] "container2_stop_0 18node1" -> "container2_delete_0 18node1" [ style = bold] "container2_stop_0 18node1" -> "container2_delete_0 18node2" [ style = bold] "container2_stop_0 18node1" -> "container2_delete_0 18node3" [ style = bold] "container2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +"lsb-dummy_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"lsb-dummy_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] "lxc-ms_clear_failcount_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = dashed] "lxc-ms_clear_failcount_0 lxc1" -> "lxc-ms_stop_0 lxc2" [ style = dashed] "lxc-ms_clear_failcount_0 lxc1" [ style=dashed color="red" fontcolor="black"] "lxc-ms_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc-ms_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc-ms_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc2" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] "lxc-ms_demote_0 lxc2" -> "lxc2_stop_0 18node1" [ style = bold] "lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node1" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node2" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node3" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] "lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node1" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node2" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node3" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc2_stop_0 18node1" [ style = bold] "lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node1" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node1" -> "container1_stop_0 18node1" [ style = bold] "lxc1_stop_0 18node1" -> "lxc1_delete_0 18node1" [ style = bold] "lxc1_stop_0 18node1" -> "lxc1_delete_0 18node2" [ style = bold] "lxc1_stop_0 18node1" -> "lxc1_delete_0 18node3" [ style = bold] "lxc1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc2_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc2_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc2_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc2_stop_0 18node1" -> "all_stopped" [ style = bold] "lxc2_stop_0 18node1" -> "container2_stop_0 18node1" [ style = bold] "lxc2_stop_0 18node1" -> "lxc2_delete_0 18node1" [ style = bold] "lxc2_stop_0 18node1" -> "lxc2_delete_0 18node2" [ style = bold] "lxc2_stop_0 18node1" -> "lxc2_delete_0 18node3" [ style = bold] "lxc2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +"migrator_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"migrator_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"ping-1_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"ping-1_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.87_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.87_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.88_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.88_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.89_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"r192.168.122.89_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"rsc_18node1_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"rsc_18node1_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"rsc_18node2_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"rsc_18node2_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] +"rsc_18node3_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"rsc_18node3_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] } diff --git a/pengine/test10/whitebox-orphan-ms.exp b/pengine/test10/whitebox-orphan-ms.exp index ef81317ba7..564e92dfcf 100644 --- a/pengine/test10/whitebox-orphan-ms.exp +++ b/pengine/test10/whitebox-orphan-ms.exp @@ -1,372 +1,372 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-orphaned.dot b/pengine/test10/whitebox-orphaned.dot index e50252d437..4ef1f7e199 100644 --- a/pengine/test10/whitebox-orphaned.dot +++ b/pengine/test10/whitebox-orphaned.dot @@ -1,39 +1,47 @@ digraph "g" { +"A_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] +"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] "B_start_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc2" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] +"C_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"D_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] +"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_clear_failcount_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "container1_clear_failcount_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "container1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" -> "container1_delete_0 18node1" [ style = bold] "container1_stop_0 18node2" -> "container1_delete_0 18node2" [ style = bold] "container1_stop_0 18node2" -> "container1_delete_0 18node3" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_clear_failcount_0 18node2" -> "lxc1_stop_0 18node2" [ style = bold] "lxc1_clear_failcount_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_delete_0 18node1" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_delete_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_delete_0 18node3" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-orphaned.exp b/pengine/test10/whitebox-orphaned.exp index 334e59452b..843f1863a7 100644 --- a/pengine/test10/whitebox-orphaned.exp +++ b/pengine/test10/whitebox-orphaned.exp @@ -1,228 +1,262 @@ - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + + + + - + - + + + + + + + + + + + + + + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-orphaned.summary b/pengine/test10/whitebox-orphaned.summary index 7d173b2375..e0928b17b0 100644 --- a/pengine/test10/whitebox-orphaned.summary +++ b/pengine/test10/whitebox-orphaned.summary @@ -1,55 +1,58 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] M (ocf::pacemaker:Dummy): ORPHANED Started lxc1 Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 container1 (ocf::heartbeat:VirtualDomain): ORPHANED Started 18node2 lxc1 (ocf::pacemaker:remote): ORPHANED Started 18node2 Transition Summary: * Stop M:4 (lxc1) due to node availability * Move B ( lxc1 -> lxc2 ) * Stop container1 (18node2) due to node availability * Stop lxc1 (18node2) due to node availability Executing cluster transition: * Pseudo action: M-clone_stop_0 - * Resource action: B stop on lxc1 + * Resource action: A monitor on lxc2 + * Resource action: B monitor on lxc2 + * Resource action: D monitor on lxc2 * Cluster action: clear_failcount for container1 on 18node2 * Cluster action: clear_failcount for lxc1 on 18node2 * Resource action: M stop on lxc1 * Pseudo action: M-clone_stopped_0 - * Resource action: B start on lxc2 + * Resource action: B stop on lxc1 * Resource action: lxc1 stop on 18node2 * Resource action: lxc1 delete on 18node3 * Resource action: lxc1 delete on 18node2 * Resource action: lxc1 delete on 18node1 - * Resource action: B monitor=10000 on lxc2 + * Resource action: B start on lxc2 * Resource action: container1 stop on 18node2 * Resource action: container1 delete on 18node3 * Resource action: container1 delete on 18node2 * Resource action: container1 delete on 18node1 * Pseudo action: all_stopped + * Resource action: B monitor=10000 on lxc2 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc2 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-start.dot b/pengine/test10/whitebox-start.dot index 8b4dbcdda3..28f747abf4 100644 --- a/pengine/test10/whitebox-start.dot +++ b/pengine/test10/whitebox-start.dot @@ -1,34 +1,38 @@ digraph "g" { +"A_monitor_0 lxc2" -> "A_start_0 lxc1" [ style = bold] +"A_monitor_0 lxc2" -> "A_stop_0 18node1" [ style = bold] +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "A_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "A_start_0 lxc1" -> "A_monitor_10000 lxc1" [ style = bold] "A_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "A_stop_0 18node1" -> "A_start_0 lxc1" [ style = bold] "A_stop_0 18node1" -> "all_stopped" [ style = bold] "A_stop_0 18node1" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 18node3" [ style=bold color="green" fontcolor="black"] "B_start_0 18node3" -> "B_monitor_10000 18node3" [ style = bold] "B_start_0 18node3" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc2" -> "B_start_0 18node3" [ style = bold] "B_stop_0 lxc2" -> "all_stopped" [ style = bold] "B_stop_0 lxc2" [ style=bold color="green" fontcolor="black"] +"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node1" -> "A_start_0 lxc1" [ style = bold] "container1_start_0 18node1" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node1" -> "lxc1_start_0 18node1" [ style = bold] "container1_start_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node1" -> "A_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "A_start_0 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "lxc1_monitor_30000 18node1" [ style = bold] "lxc1_start_0 18node1" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-start.exp b/pengine/test10/whitebox-start.exp index ccccf60210..360b8a7dc3 100644 --- a/pengine/test10/whitebox-start.exp +++ b/pengine/test10/whitebox-start.exp @@ -1,189 +1,214 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + + + + + - + + + + + + + + + + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-start.summary b/pengine/test10/whitebox-start.summary index e76e28a45f..01a1b74641 100644 --- a/pengine/test10/whitebox-start.summary +++ b/pengine/test10/whitebox-start.summary @@ -1,53 +1,55 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Stopped container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] Stopped: [ lxc1 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc2 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Start container1 (18node1) * Start M:4 (lxc1) * Move A ( 18node1 -> lxc1 ) * Move B ( lxc2 -> 18node3 ) * Start lxc1 (18node1) Executing cluster transition: * Resource action: container1 start on 18node1 * Pseudo action: M-clone_start_0 - * Resource action: A stop on 18node1 + * Resource action: A monitor on lxc2 * Resource action: B stop on lxc2 + * Resource action: D monitor on lxc2 * Resource action: lxc1 start on 18node1 - * Pseudo action: all_stopped * Resource action: M start on lxc1 * Pseudo action: M-clone_running_0 - * Resource action: A start on lxc1 + * Resource action: A stop on 18node1 * Resource action: B start on 18node3 * Resource action: lxc1 monitor=30000 on 18node1 + * Pseudo action: all_stopped * Resource action: M monitor=10000 on lxc1 - * Resource action: A monitor=10000 on lxc1 + * Resource action: A start on lxc1 * Resource action: B monitor=10000 on 18node3 + * Resource action: A monitor=10000 on lxc1 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node1 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started lxc1 B (ocf::pacemaker:Dummy): Started 18node3 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-stop.dot b/pengine/test10/whitebox-stop.dot index 9900483098..0ecdcba2ad 100644 --- a/pengine/test10/whitebox-stop.dot +++ b/pengine/test10/whitebox-stop.dot @@ -1,23 +1,28 @@ digraph "g" { +"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] +"B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] +"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] +"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] "B_start_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc2" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] +"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] } diff --git a/pengine/test10/whitebox-stop.exp b/pengine/test10/whitebox-stop.exp index 354b6d4bf1..84a5288316 100644 --- a/pengine/test10/whitebox-stop.exp +++ b/pengine/test10/whitebox-stop.exp @@ -1,127 +1,161 @@ - + - + - + - + - + - + - + - + - + + + + + + + + + + - + - + - + - + + + + - + - + + + + + + + + + + + + + + - + - + + + + + + + + + + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-stop.summary b/pengine/test10/whitebox-stop.summary index 9b15ea0c60..44e570fc30 100644 --- a/pengine/test10/whitebox-stop.summary +++ b/pengine/test10/whitebox-stop.summary @@ -1,48 +1,51 @@ 1 of 14 resources DISABLED and 0 BLOCKED from being started due to failures Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 ( disabled ) container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Stop container1 (18node2) due to node availability * Stop M:4 (lxc1) due to node availability * Move B ( lxc1 -> lxc2 ) * Stop lxc1 (18node2) due to node availability Executing cluster transition: * Pseudo action: M-clone_stop_0 - * Resource action: B stop on lxc1 + * Resource action: A monitor on lxc2 + * Resource action: B monitor on lxc2 + * Resource action: D monitor on lxc2 * Resource action: M stop on lxc1 * Pseudo action: M-clone_stopped_0 - * Resource action: B start on lxc2 + * Resource action: B stop on lxc1 * Resource action: lxc1 stop on 18node2 * Resource action: container1 stop on 18node2 - * Resource action: B monitor=10000 on lxc2 + * Resource action: B start on lxc2 * Pseudo action: all_stopped + * Resource action: B monitor=10000 on lxc2 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Stopped ( disabled ) container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] Stopped: [ lxc1 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc2 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-unexpectedly-running.dot b/pengine/test10/whitebox-unexpectedly-running.dot index 0e2e5bb62a..fa1171ec83 100644 --- a/pengine/test10/whitebox-unexpectedly-running.dot +++ b/pengine/test10/whitebox-unexpectedly-running.dot @@ -1,20 +1,26 @@ digraph "g" { +"FAKE-crashed_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] +"FAKE-crashed_start_0 18builder" -> "FAKE-crashed_monitor_60000 18builder" [ style = bold] +"FAKE-crashed_start_0 18builder" -> "remote2_start_0 18builder" [ style = bold] +"FAKE-crashed_start_0 18builder" [ style=bold color="green" fontcolor="black"] +"FAKE-crashed_stop_0 18builder" -> "FAKE-crashed_start_0 18builder" [ style = bold] +"FAKE-crashed_stop_0 18builder" -> "all_stopped" [ style = bold] +"FAKE-crashed_stop_0 18builder" -> "stonith 'reboot' remote2" [ style = bold] +"FAKE-crashed_stop_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] -"FAKE_start_0 18builder" -> "FAKE_monitor_60000 18builder" [ style = bold] -"FAKE_start_0 18builder" -> "remote1_start_0 18builder" [ style = bold] -"FAKE_start_0 18builder" [ style=bold color="green" fontcolor="black"] -"FAKE_stop_0 18builder" -> "FAKE_start_0 18builder" [ style = bold] -"FAKE_stop_0 18builder" -> "all_stopped" [ style = bold] -"FAKE_stop_0 18builder" -> "stonith 'reboot' remote1" [ style = bold] -"FAKE_stop_0 18builder" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "remote1_monitor_0 18builder" -> "remote1_start_0 18builder" [ style = bold] "remote1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] "remote1_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] "remote1_start_0 18builder" -> "remote1_monitor_30000 18builder" [ style = bold] "remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"] -"stonith 'reboot' remote1" -> "stonith_complete" [ style = bold] -"stonith 'reboot' remote1" [ style=bold color="green" fontcolor="orange"] +"remote2_monitor_0 18builder" -> "remote2_start_0 18builder" [ style = bold] +"remote2_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] +"remote2_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] +"remote2_start_0 18builder" -> "remote2_monitor_30000 18builder" [ style = bold] +"remote2_start_0 18builder" [ style=bold color="green" fontcolor="black"] +"stonith 'reboot' remote2" -> "stonith_complete" [ style = bold] +"stonith 'reboot' remote2" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-unexpectedly-running.exp b/pengine/test10/whitebox-unexpectedly-running.exp index a0e5caed79..c4e13b93c0 100644 --- a/pengine/test10/whitebox-unexpectedly-running.exp +++ b/pengine/test10/whitebox-unexpectedly-running.exp @@ -1,117 +1,161 @@ - + + + + + + + + + + - + - + - - - + + + - + - + - - - + + + - + - + - + - + - + - - - - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/whitebox-unexpectedly-running.scores b/pengine/test10/whitebox-unexpectedly-running.scores index 7f8a1d91c5..45e1d397b9 100644 --- a/pengine/test10/whitebox-unexpectedly-running.scores +++ b/pengine/test10/whitebox-unexpectedly-running.scores @@ -1,5 +1,13 @@ Allocation scores: native_color: FAKE allocation score on 18builder: 0 native_color: FAKE allocation score on remote1: -INFINITY +native_color: FAKE allocation score on remote2: -INFINITY +native_color: FAKE-crashed allocation score on 18builder: 0 +native_color: FAKE-crashed allocation score on remote1: -INFINITY +native_color: FAKE-crashed allocation score on remote2: -INFINITY native_color: remote1 allocation score on 18builder: 0 native_color: remote1 allocation score on remote1: -INFINITY +native_color: remote1 allocation score on remote2: -INFINITY +native_color: remote2 allocation score on 18builder: 0 +native_color: remote2 allocation score on remote1: -INFINITY +native_color: remote2 allocation score on remote2: -INFINITY diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary index eef4f6353e..3e0c89870f 100644 --- a/pengine/test10/whitebox-unexpectedly-running.summary +++ b/pengine/test10/whitebox-unexpectedly-running.summary @@ -1,28 +1,35 @@ Current cluster status: Online: [ 18builder ] - FAKE (ocf::pacemaker:Dummy): FAILED 18builder + FAKE (ocf::pacemaker:Dummy): Started 18builder + FAKE-crashed (ocf::pacemaker:Dummy): FAILED 18builder Transition Summary: - * Fence (reboot) remote1 (resource: FAKE) 'guest is unclean' - * Recover FAKE ( 18builder ) + * Fence (reboot) remote2 (resource: FAKE-crashed) 'guest is unclean' + * Recover FAKE-crashed ( 18builder ) * Start remote1 (18builder) + * Start remote2 ( 18builder ) Executing cluster transition: - * Resource action: FAKE stop on 18builder + * Resource action: FAKE monitor=60000 on 18builder + * Resource action: FAKE-crashed stop on 18builder * Resource action: remote1 monitor on 18builder - * Pseudo action: stonith-remote1-reboot on remote1 + * Resource action: remote2 monitor on 18builder + * Pseudo action: stonith-remote2-reboot on remote2 * Pseudo action: stonith_complete * Pseudo action: all_stopped - * Resource action: FAKE start on 18builder + * Resource action: FAKE-crashed start on 18builder * Resource action: remote1 start on 18builder - * Resource action: FAKE monitor=60000 on 18builder + * Resource action: remote2 start on 18builder + * Resource action: FAKE-crashed monitor=60000 on 18builder * Resource action: remote1 monitor=30000 on 18builder + * Resource action: remote2 monitor=30000 on 18builder Revised cluster status: Online: [ 18builder ] -Containers: [ remote1:FAKE ] +Containers: [ remote1:FAKE remote2:FAKE-crashed ] FAKE (ocf::pacemaker:Dummy): Started 18builder + FAKE-crashed (ocf::pacemaker:Dummy): Started 18builder diff --git a/pengine/test10/whitebox-unexpectedly-running.xml b/pengine/test10/whitebox-unexpectedly-running.xml index 4ec20c472c..638ed1a295 100644 --- a/pengine/test10/whitebox-unexpectedly-running.xml +++ b/pengine/test10/whitebox-unexpectedly-running.xml @@ -1,44 +1,55 @@ + + + + + + + + + + +