diff --git a/crm/pengine/group.c b/crm/pengine/group.c index ae4c22fc2e..de5f9358d4 100644 --- a/crm/pengine/group.c +++ b/crm/pengine/group.c @@ -1,474 +1,481 @@ -/* $Id: group.c,v 1.67 2006/06/22 13:32:58 andrew Exp $ */ +/* $Id: group.c,v 1.68 2006/07/03 11:47:38 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include extern gboolean rsc_colocation_new( const char *id, enum con_strength strength, resource_t *rsc_lh, resource_t *rsc_rh, const char *state_lh, const char *state_rh); typedef struct group_variant_data_s { int num_children; GListPtr child_list; /* resource_t* */ resource_t *self; resource_t *first_child; resource_t *last_child; gboolean colocated; gboolean ordered; gboolean child_starting; gboolean child_stopping; } group_variant_data_t; #define get_group_variant_data(data, rsc) \ CRM_ASSERT(rsc != NULL); \ CRM_ASSERT(rsc->variant == pe_group); \ CRM_ASSERT(rsc->variant_opaque != NULL); \ data = (group_variant_data_t *)rsc->variant_opaque; \ void group_assign_color(resource_t *rsc, color_t *group_color); void group_set_cmds(resource_t *rsc) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); group_data->self->cmds = &resource_class_alloc_functions[group_data->self->variant]; slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds = &resource_class_alloc_functions[child_rsc->variant]; child_rsc->cmds->set_cmds(child_rsc); ); } int group_num_allowed_nodes(resource_t *rsc) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); if(group_data->colocated == FALSE) { pe_config_err("Cannot clone non-colocated group: %s", rsc->id); return 0; } return group_data->self->cmds->num_allowed_nodes(group_data->self); } color_t * group_color(resource_t *rsc, pe_working_set_t *data_set) { color_t *group_color = NULL; group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Coloring children of: %s", rsc->id); + slist_iter( + coloc, rsc_colocation_t, rsc->rsc_cons, lpc, + crm_debug_3("Pre-Processing %s for %s", coloc->id, rsc->id); + group_data->first_child->cmds->rsc_colocation_lh( + group_data->first_child, coloc->rsc_rh, coloc); + ); + slist_iter( child_rsc, resource_t, group_data->child_list, lpc, group_color = child_rsc->cmds->color(child_rsc, data_set); CRM_CHECK(group_color != NULL, continue); native_assign_color(rsc, group_color); ); return group_color; } void group_assign_color(resource_t *rsc, color_t *group_color) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Coloring children of: %s", rsc->id); CRM_CHECK(group_color != NULL, return); native_assign_color(rsc, group_color); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, native_assign_color(child_rsc, group_color); ); } void group_update_pseudo_status(resource_t *parent, resource_t *child); void group_create_actions(resource_t *rsc, pe_working_set_t *data_set) { action_t *op = NULL; group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->create_actions(child_rsc, data_set); group_update_pseudo_status(rsc, child_rsc); ); op = start_action(group_data->self, NULL, !group_data->child_starting); op->pseudo = TRUE; op = custom_action(group_data->self, started_key(group_data->self), CRMD_ACTION_STARTED, NULL, !group_data->child_starting, TRUE, data_set); op->pseudo = TRUE; op = stop_action(group_data->self, NULL, !group_data->child_stopping); op->pseudo = TRUE; op = custom_action(group_data->self, stopped_key(group_data->self), CRMD_ACTION_STOPPED, NULL, !group_data->child_stopping, TRUE, data_set); op->pseudo = TRUE; rsc->actions = group_data->self->actions; } void group_update_pseudo_status(resource_t *parent, resource_t *child) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, parent); if(group_data->child_stopping && group_data->child_starting) { return; } slist_iter( action, action_t, child->actions, lpc, if(action->optional) { continue; } if(safe_str_eq(CRMD_ACTION_STOP, action->task) && action->runnable) { group_data->child_stopping = TRUE; } else if(safe_str_eq(CRMD_ACTION_START, action->task) && action->runnable) { group_data->child_starting = TRUE; } ); } void group_internal_constraints(resource_t *rsc, pe_working_set_t *data_set) { resource_t *last_rsc = NULL; group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); group_data->self->cmds->internal_constraints(group_data->self, data_set); custom_action_order( group_data->self, stopped_key(group_data->self), NULL, group_data->self, start_key(group_data->self), NULL, pe_ordering_optional, data_set); custom_action_order( group_data->self, stop_key(group_data->self), NULL, group_data->self, stopped_key(group_data->self), NULL, pe_ordering_optional, data_set); custom_action_order( group_data->self, start_key(group_data->self), NULL, group_data->self, started_key(group_data->self), NULL, pe_ordering_optional, data_set); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->internal_constraints(child_rsc, data_set); if(group_data->colocated && child_rsc != group_data->first_child) { rsc_colocation_new( "group:internal_colocation", pecs_must, group_data->first_child, child_rsc, NULL, NULL); } if(group_data->ordered == FALSE) { order_start_start( group_data->self, child_rsc, pe_ordering_optional); custom_action_order( child_rsc, start_key(child_rsc), NULL, group_data->self, started_key(group_data->self), NULL, pe_ordering_optional, data_set); order_stop_stop( group_data->self, child_rsc, pe_ordering_optional); custom_action_order( child_rsc, stop_key(child_rsc), NULL, group_data->self, stopped_key(group_data->self), NULL, pe_ordering_optional, data_set); continue; } if(last_rsc != NULL) { order_start_start( last_rsc, child_rsc, pe_ordering_optional); order_stop_stop( child_rsc, last_rsc, pe_ordering_optional); /* recovery */ child_rsc->restart_type = pe_restart_restart; order_start_start( last_rsc, child_rsc, pe_ordering_recover); order_stop_stop( child_rsc, last_rsc, pe_ordering_recover); } else { custom_action_order( child_rsc, stop_key(child_rsc), NULL, group_data->self, stopped_key(group_data->self), NULL, pe_ordering_optional, data_set); order_start_start(group_data->self, child_rsc, pe_ordering_optional); } last_rsc = child_rsc; ); if(group_data->ordered && last_rsc != NULL) { custom_action_order( last_rsc, start_key(last_rsc), NULL, group_data->self, started_key(group_data->self), NULL, pe_ordering_optional, data_set); order_stop_stop( group_data->self, last_rsc, pe_ordering_optional); } } void group_rsc_colocation_lh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { group_variant_data_t *group_data = NULL; if(rsc_lh == NULL) { pe_err("rsc_lh was NULL for %s", constraint->id); return; } else if(rsc_rh == NULL) { pe_err("rsc_rh was NULL for %s", constraint->id); return; } crm_debug_4("Processing constraints from %s", rsc_lh->id); get_group_variant_data(group_data, rsc_lh); if(group_data->colocated) { group_data->first_child->cmds->rsc_colocation_lh( group_data->first_child, rsc_rh, constraint); return; } if(constraint->strength != pecs_must_not) { pe_config_err("Cannot colocate resources with" " non-colocated group: %s", rsc_lh->id); return; } slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->rsc_colocation_lh( child_rsc, rsc_rh, constraint); ); } void group_rsc_colocation_rh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc_rh); CRM_CHECK(rsc_lh->variant == pe_native, return); crm_debug_3("Processing RH of constraint %s", constraint->id); print_resource(LOG_DEBUG_3, "LHS", rsc_lh, TRUE); if(group_data->colocated) { group_data->first_child->cmds->rsc_colocation_rh( rsc_lh, group_data->first_child, constraint); return; } if(constraint->strength != pecs_must_not) { pe_config_err("Cannot colocate resources with" " non-colocated group: %s", rsc_rh->id); return; } slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->rsc_colocation_rh( rsc_lh, child_rsc, constraint); ); } void group_rsc_order_lh(resource_t *rsc, order_constraint_t *order) { char *stop_id = NULL; char *start_id = NULL; group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Processing LH of ordering constraint %d", order->id); if(group_data->self == NULL) { return; } stop_id = stop_key(group_data->self); start_id = start_key(group_data->self); if(safe_str_eq(order->lh_action_task, start_id)) { crm_free(order->lh_action_task); order->lh_action_task = started_key(group_data->self); } else if(safe_str_eq(order->lh_action_task, stop_id)) { crm_free(order->lh_action_task); order->lh_action_task = stopped_key(group_data->self); } crm_free(start_id); crm_free(stop_id); group_data->self->cmds->rsc_order_lh(group_data->self, order); } void group_rsc_order_rh( action_t *lh_action, resource_t *rsc, order_constraint_t *order) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Processing RH of ordering constraint %d", order->id); if(group_data->self == NULL) { return; } group_data->self->cmds->rsc_order_rh(lh_action, group_data->self, order); } void group_rsc_location(resource_t *rsc, rsc_to_node_t *constraint) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Processing actions from %s", group_data->self->id); group_data->self->cmds->rsc_location(group_data->self, constraint); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->rsc_location(child_rsc, constraint); ); } void group_expand(resource_t *rsc, pe_working_set_t *data_set) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); crm_debug_3("Processing actions from %s", rsc->id); CRM_CHECK(group_data->self != NULL, return); group_data->self->cmds->expand(group_data->self, data_set); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->expand(child_rsc, data_set); ); } void group_agent_constraints(resource_t *rsc) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->agent_constraints(child_rsc); ); } void group_create_notify_element(resource_t *rsc, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->create_notify_element( child_rsc, op, n_data, data_set); ); } gboolean group_create_probe(resource_t *rsc, node_t *node, action_t *complete, gboolean force, pe_working_set_t *data_set) { gboolean any_created = FALSE; group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, any_created = child_rsc->cmds->create_probe( child_rsc, node, complete, force, data_set) || any_created; ); return any_created; } void group_stonith_ordering( resource_t *rsc, action_t *stonith_op, pe_working_set_t *data_set) { group_variant_data_t *group_data = NULL; get_group_variant_data(group_data, rsc); slist_iter( child_rsc, resource_t, group_data->child_list, lpc, child_rsc->cmds->stonith_ordering( child_rsc, stonith_op, data_set); ); } diff --git a/crm/pengine/native.c b/crm/pengine/native.c index 3b3e4462db..6e1f42bf14 100644 --- a/crm/pengine/native.c +++ b/crm/pengine/native.c @@ -1,1795 +1,1797 @@ -/* $Id: native.c,v 1.154 2006/06/22 13:33:34 andrew Exp $ */ +/* $Id: native.c,v 1.155 2006/07/03 11:47:38 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #define DELETE_THEN_REFRESH 1 extern color_t *add_color(resource_t *rh_resource, color_t *color); gboolean native_choose_color(resource_t *lh_resource, color_t *no_color); void native_update_node_weight(resource_t *rsc, rsc_to_node_t *cons, node_t *cons_node, GListPtr nodes); void native_rsc_colocation_rh_must(resource_t *rsc_lh, gboolean update_lh, resource_t *rsc_rh, gboolean update_rh); void native_rsc_colocation_rh_mustnot(resource_t *rsc_lh, gboolean update_lh, resource_t *rsc_rh, gboolean update_rh); void filter_nodes(resource_t *rsc); int num_allowed_nodes4color(color_t *color); void create_notifications(resource_t *rsc, pe_working_set_t *data_set); void Recurring(resource_t *rsc, action_t *start, node_t *node, pe_working_set_t *data_set); void pe_pre_notify( resource_t *rsc, node_t *node, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set); void pe_post_notify( resource_t *rsc, node_t *node, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set); gboolean DeleteRsc(resource_t *rsc, node_t *node, pe_working_set_t *data_set); void NoRoleChange(resource_t *rsc, node_t *current, node_t *next, pe_working_set_t *data_set); gboolean StopRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set); gboolean StartRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set); extern gboolean DemoteRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set); gboolean PromoteRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set); gboolean RoleError(resource_t *rsc, node_t *next, pe_working_set_t *data_set); gboolean NullOp(resource_t *rsc, node_t *next, pe_working_set_t *data_set); enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = { /* Current State */ /* Next State: Unknown Stopped Started Slave Master */ /* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, }, /* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, }, /* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, /* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, /* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, }, }; gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t*,node_t*,pe_working_set_t*) = { /* Current State */ /* Next State: Unknown Stopped Started Slave Master */ /* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, }, /* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, }, /* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, }, /* Slave */ { RoleError, StopRsc, RoleError, NullOp, PromoteRsc, }, /* Master */ { RoleError, RoleError, RoleError, DemoteRsc, NullOp, }, }; typedef struct native_variant_data_s { /* GListPtr allowed_nodes; /\* node_t* *\/ */ } native_variant_data_t; #define get_native_variant_data(data, rsc) \ CRM_ASSERT(rsc->variant == pe_native); \ CRM_ASSERT(rsc->variant_opaque != NULL); \ data = (native_variant_data_t *)rsc->variant_opaque; void native_set_cmds(resource_t *rsc) { } int native_num_allowed_nodes(resource_t *rsc) { int num_nodes = 0; if(rsc->next_role == RSC_ROLE_STOPPED) { return 0; } if(rsc->color) { crm_debug_4("Colored case"); num_nodes = num_allowed_nodes4color(rsc->color); } else if(rsc->candidate_colors) { /* TODO: sort colors first */ color_t *color = g_list_nth_data(rsc->candidate_colors, 0); crm_debug_4("Candidate colors case"); num_nodes = num_allowed_nodes4color(color); } else { crm_debug_4("Default case"); slist_iter( this_node, node_t, rsc->allowed_nodes, lpc, crm_debug_3("Rsc %s Checking %s: %d", rsc->id, this_node->details->uname, this_node->weight); if(this_node->details->shutdown || this_node->details->online == FALSE) { this_node->weight = -INFINITY; } if(this_node->weight < 0) { continue; /* } else if(this_node->details->unclean) { */ /* continue; */ } num_nodes++; ); } crm_debug_2("Resource %s can run on %d nodes", rsc->id, num_nodes); return num_nodes; } int num_allowed_nodes4color(color_t *color) { int num_nodes = 0; if(color->details->pending == FALSE) { if(color->details->chosen_node) { return 1; } return 0; } slist_iter( this_node, node_t, color->details->candidate_nodes, lpc, crm_debug_3("Checking %s: %d", this_node->details->uname, this_node->weight); if(this_node->details->shutdown || this_node->details->online == FALSE) { this_node->weight = -INFINITY; } if(this_node->weight < 0) { continue; /* } else if(this_node->details->unclean) { */ /* continue; */ } num_nodes++; ); return num_nodes; } color_t * native_color(resource_t *rsc, pe_working_set_t *data_set) { color_t *new_color = NULL; print_resource(LOG_DEBUG_2, "Coloring: ", rsc, FALSE); if(rsc->provisional == FALSE) { return rsc->color; } rsc->rsc_cons = g_list_sort( rsc->rsc_cons, sort_cons_strength); /*------ Pre-processing */ slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons, lpc, crm_debug_3("Pre-Processing %s", constraint->id); rsc->cmds->rsc_colocation_lh( rsc, constraint->rsc_rh, constraint); ); if( native_choose_color(rsc, data_set->no_color) ) { crm_debug_3("Colored resource %s with color %d", rsc->id, rsc->color->id); new_color = rsc->color; } else { if(rsc->allowed_nodes != NULL) { /* filter out nodes with a negative weight */ filter_nodes(rsc); new_color = create_color(data_set, rsc, rsc->allowed_nodes); native_assign_color(rsc, new_color); + crm_debug_3("Colored resource %s with new color %d", + rsc->id, rsc->color->id); } if(new_color == NULL) { pe_warn("Resource %s cannot run anywhere", rsc->id); print_resource(LOG_ERR, "No color: ", rsc, FALSE); native_assign_color(rsc, data_set->no_color); new_color = data_set->no_color; } } rsc->provisional = FALSE; /*------ Post-processing */ #if 1 slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons, lpc, crm_debug_3("Post-Processing %s", constraint->id); rsc->cmds->rsc_colocation_lh( rsc, constraint->rsc_rh, constraint); ); #endif - print_resource(LOG_DEBUG_3, "Colored", rsc, TRUE); + print_resource(LOG_DEBUG_3, "Colored ", rsc, TRUE); return new_color; } void Recurring(resource_t *rsc, action_t *start, node_t *node, pe_working_set_t *data_set) { char *key = NULL; const char *name = NULL; const char *value = NULL; const char *interval = NULL; const char *node_uname = NULL; int interval_ms = 0; action_t *mon = NULL; gboolean is_optional = TRUE; GListPtr possible_matches = NULL; crm_debug_2("Creating recurring actions for %s", rsc->id); if(node != NULL) { node_uname = node->details->uname; } xml_child_iter_filter( rsc->ops_xml, operation, "op", is_optional = TRUE; name = crm_element_value(operation, "name"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); interval_ms = crm_get_msec(interval); if(interval_ms <= 0) { continue; } value = crm_element_value(operation, "disabled"); if(crm_is_true(value)) { continue; } key = generate_op_key(rsc->id, name, interval_ms); if(start != NULL) { crm_debug_3("Marking %s %s due to %s", key, start->optional?"optional":"manditory", start->uuid); is_optional = start->optional; } else { crm_debug_2("Marking %s optional", key); is_optional = TRUE; } /* start a monitor for an already active resource */ possible_matches = find_actions_exact(rsc->actions, key, node); if(possible_matches == NULL) { is_optional = FALSE; crm_debug_3("Marking %s manditory: not active", key); } value = crm_element_value(operation, "role"); if((rsc->next_role == RSC_ROLE_MASTER && value == NULL) || (value != NULL && text2role(value) != rsc->next_role)) { int log_level = LOG_DEBUG_2; const char *foo = "Ignoring"; if(is_optional) { log_level = LOG_INFO; foo = "Cancelling"; /* its running : cancel it */ mon = custom_action( rsc, crm_strdup(key), CRMD_ACTION_CANCEL, node, FALSE, TRUE, data_set); mon->task = CRMD_ACTION_CANCEL; add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval); add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name); custom_action_order( rsc, NULL, mon, rsc, promote_key(rsc), NULL, pe_ordering_optional, data_set); mon = NULL; } crm_log_maybe(log_level, "%s action %s (%s vs. %s)", foo , key, value?value:role2text(RSC_ROLE_SLAVE), role2text(rsc->next_role)); crm_free(key); key = NULL; continue; } mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set); if(is_optional) { crm_debug("%s\t %s (optional)", crm_str(node_uname), mon->uuid); } if(start == NULL || start->runnable == FALSE) { crm_debug("%s\t %s (cancelled : start un-runnable)", crm_str(node_uname), mon->uuid); mon->runnable = FALSE; } else if(node == NULL || node->details->online == FALSE || node->details->unclean) { crm_debug("%s\t %s (cancelled : no node available)", crm_str(node_uname), mon->uuid); mon->runnable = FALSE; } else if(mon->optional == FALSE) { crm_notice("%s\t %s", crm_str(node_uname),mon->uuid); } custom_action_order(rsc, start_key(rsc), NULL, NULL, crm_strdup(key), mon, pe_ordering_restart, data_set); if(rsc->next_role == RSC_ROLE_MASTER) { char *running_master = crm_itoa(EXECRA_RUNNING_MASTER); add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master); custom_action_order( rsc, promote_key(rsc), NULL, rsc, NULL, mon, pe_ordering_optional, data_set); crm_free(running_master); } ); } void native_create_actions(resource_t *rsc, pe_working_set_t *data_set) { action_t *start = NULL; node_t *chosen = NULL; enum rsc_role_e role = RSC_ROLE_UNKNOWN; enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; CRM_CHECK(rsc->color != NULL, return); chosen = rsc->color->details->chosen_node; if(chosen != NULL) { CRM_CHECK(rsc->next_role != RSC_ROLE_UNKNOWN, rsc->next_role = RSC_ROLE_STARTED); } unpack_instance_attributes( rsc->xml, XML_TAG_ATTR_SETS, chosen?chosen->details->attrs:NULL, rsc->parameters, NULL, data_set->now); crm_debug("%s: %s->%s", rsc->id, role2text(rsc->role), role2text(rsc->next_role)); if(g_list_length(rsc->running_on) > 1) { if(rsc->recovery_type == recovery_stop_start) { pe_err("Attempting recovery of resource %s", rsc->id); StopRsc(rsc, NULL, data_set); rsc->role = RSC_ROLE_STOPPED; } } else if(rsc->running_on != NULL) { node_t *current = rsc->running_on->data; NoRoleChange(rsc, current, chosen, data_set); } else if(rsc->role == RSC_ROLE_STOPPED && rsc->next_role == RSC_ROLE_STOPPED) { char *key = start_key(rsc); GListPtr possible_matches = find_actions(rsc->actions, key, NULL); slist_iter( action, action_t, possible_matches, lpc, action->optional = TRUE; /* action->pseudo = TRUE; */ ); crm_debug_2("Stopping a stopped resource"); crm_free(key); return; } role = rsc->role; while(role != rsc->next_role) { next_role = rsc_state_matrix[role][rsc->next_role]; crm_debug_2("Executing: %s->%s (%s)", role2text(role), role2text(next_role), rsc->id); if(rsc_action_matrix[role][next_role]( rsc, chosen, data_set) == FALSE) { break; } role = next_role; } if(rsc->next_role != RSC_ROLE_STOPPED && rsc->is_managed) { start = start_action(rsc, chosen, TRUE); Recurring(rsc, start, chosen, data_set); } } void native_internal_constraints(resource_t *rsc, pe_working_set_t *data_set) { order_restart(rsc); custom_action_order(rsc, demote_key(rsc), NULL, rsc, stop_key(rsc), NULL, pe_ordering_manditory, data_set); custom_action_order(rsc, start_key(rsc), NULL, rsc, promote_key(rsc), NULL, pe_ordering_optional, data_set); custom_action_order( rsc, stop_key(rsc), NULL, rsc, delete_key(rsc), NULL, pe_ordering_optional, data_set); custom_action_order( rsc, delete_key(rsc), NULL, rsc, start_key(rsc), NULL, pe_ordering_manditory, data_set); } void native_rsc_colocation_lh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { if(rsc_lh == NULL) { pe_err("rsc_lh was NULL for %s", constraint->id); return; } else if(constraint->rsc_rh == NULL) { pe_err("rsc_rh was NULL for %s", constraint->id); return; } crm_debug_2("Processing colocation constraint between %s and %s", rsc_lh->id, rsc_rh->id); rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint); } static gboolean filter_colocation_constraint( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { if(constraint->strength == pecs_ignore || constraint->strength == pecs_startstop){ crm_debug_4("Skipping constraint type %d", constraint->strength); return FALSE; } if(constraint->state_lh != NULL && text2role(constraint->state_lh) != rsc_lh->next_role) { crm_debug_4("RH: Skipping constraint: \"%s\" state filter", constraint->state_rh); return FALSE; } if(constraint->state_rh != NULL && text2role(constraint->state_rh) != rsc_rh->next_role) { crm_debug_4("RH: Skipping constraint: \"%s\" state filter", constraint->state_rh); return FALSE; } return TRUE; } void native_rsc_colocation_rh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { gboolean do_check = FALSE; gboolean update_lh = FALSE; gboolean update_rh = FALSE; crm_debug_2("%sColocating %s with %s (%s)", constraint->strength == pecs_must?"":"Anti-", rsc_lh->id, rsc_rh->id, constraint->id); if(filter_colocation_constraint(rsc_lh, rsc_rh, constraint) == FALSE) { return; } if(rsc_lh->provisional && rsc_rh->provisional) { if(constraint->strength == pecs_must) { /* update effective_priorities */ crm_debug_3("Priority update"); native_rsc_colocation_rh_must( rsc_lh, update_lh, rsc_rh, update_rh); } else { /* nothing */ crm_debug_4( "Skipping constraint, both sides provisional"); } return; } else if( (!rsc_lh->provisional) && (!rsc_rh->provisional) && (!rsc_lh->color->details->pending) && (!rsc_rh->color->details->pending) ) { /* error check */ do_check = TRUE; if(rsc_lh->effective_priority < rsc_rh->effective_priority) { update_lh = TRUE; } else if(rsc_lh->effective_priority > rsc_rh->effective_priority) { update_rh = TRUE; } else { update_lh = TRUE; update_rh = TRUE; } } else if(rsc_lh->provisional == FALSE && rsc_lh->color->details->pending == FALSE) { /* update _them_ : postproc color version */ update_rh = TRUE; } else if(rsc_rh->provisional == FALSE && rsc_rh->color->details->pending == FALSE) { /* update _us_ : postproc color alt version */ update_lh = TRUE; } else if(rsc_lh->provisional == FALSE) { /* update _them_ : preproc version */ update_rh = TRUE; } else if(rsc_rh->provisional == FALSE) { /* update _us_ : postproc version */ update_lh = TRUE; } else { pe_warn("Un-expected combination of inputs"); return; } if(update_lh) { crm_debug_4("Updating LHS"); } if(update_rh) { crm_debug_4("Updating RHS"); } if(do_check) { if(native_constraint_violated( rsc_lh, rsc_rh, constraint) == FALSE) { crm_debug_4("Constraint satisfied"); return; } /* else constraint cant be satisified */ pe_warn("Constraint %s could not be satisfied", constraint->id); if(update_lh) { pe_warn("Marking resource %s unrunnable as a result", rsc_lh->id); resource_location(rsc_lh, NULL, -INFINITY, constraint->id, NULL); if(rsc_lh->color) { crm_free(rsc_lh->color->details->chosen_node); rsc_lh->color->details->chosen_node = NULL; pe_free_shallow_adv( rsc_lh->color->details->candidate_nodes, TRUE); rsc_lh->color->details->candidate_nodes = NULL; } } if(update_rh) { pe_warn("Marking resource %s unrunnable as a result", rsc_rh->id); resource_location(rsc_rh, NULL, -INFINITY, constraint->id, NULL); if(rsc_rh->color) { crm_free(rsc_rh->color->details->chosen_node); rsc_rh->color->details->chosen_node = NULL; pe_free_shallow_adv( rsc_rh->color->details->candidate_nodes, TRUE); rsc_rh->color->details->candidate_nodes = NULL; } } } if(constraint->strength == pecs_must) { native_rsc_colocation_rh_must( rsc_lh, update_lh, rsc_rh, update_rh); return; } else if(constraint->strength != pecs_must_not) { /* unknown type */ pe_err("Unknown constraint type %d", constraint->strength); return; } native_rsc_colocation_rh_mustnot(rsc_lh, update_lh,rsc_rh, update_rh); } void native_rsc_order_lh(resource_t *lh_rsc, order_constraint_t *order) { GListPtr lh_actions = NULL; action_t *lh_action = order->lh_action; crm_debug_3("Processing LH of ordering constraint %d", order->id); if(lh_action != NULL) { lh_actions = g_list_append(NULL, lh_action); } else if(lh_action == NULL && lh_rsc != NULL) { lh_actions = find_actions( lh_rsc->actions, order->lh_action_task, NULL); if(lh_actions == NULL) { crm_debug_4("No LH-Side (%s/%s) found for constraint", lh_rsc->id, order->lh_action_task); if(lh_rsc->next_role == RSC_ROLE_STOPPED) { resource_t *rh_rsc = order->rh_rsc; if(order->rh_action && order->type == pe_ordering_restart) { crm_debug("No LH(%s/%s) found for RH(%s)...", lh_rsc->id, order->lh_action_task, order->rh_action->uuid); order->rh_action->runnable = FALSE; return; } else if(rh_rsc != NULL) { crm_debug("No LH(%s/%s) found for RH(%s/%s)...", lh_rsc->id, order->lh_action_task, rh_rsc->id, order->rh_action_task); rh_rsc->cmds->rsc_order_rh(NULL, rh_rsc, order); return; } } return; } } else { pe_warn("No LH-Side (%s) specified for constraint", order->lh_action_task); if(order->rh_rsc != NULL) { crm_debug_4("RH-Side was: (%s/%s)", order->rh_rsc->id, order->rh_action_task); } else if(order->rh_action != NULL && order->rh_action->rsc != NULL) { crm_debug_4("RH-Side was: (%s/%s)", order->rh_action->rsc->id, order->rh_action_task); } else if(order->rh_action != NULL) { crm_debug_4("RH-Side was: %s", order->rh_action_task); } else { crm_debug_4("RH-Side was NULL"); } return; } slist_iter( lh_action_iter, action_t, lh_actions, lpc, resource_t *rh_rsc = order->rh_rsc; if(rh_rsc == NULL && order->rh_action) { rh_rsc = order->rh_action->rsc; } if(rh_rsc) { rh_rsc->cmds->rsc_order_rh( lh_action_iter, rh_rsc, order); } else if(order->rh_action) { order_actions(lh_action_iter, order->rh_action, order->type); } ); pe_free_shallow_adv(lh_actions, FALSE); } void native_rsc_order_rh( action_t *lh_action, resource_t *rsc, order_constraint_t *order) { GListPtr rh_actions = NULL; action_t *rh_action = order->rh_action; crm_debug_3("Processing RH of ordering constraint %d", order->id); if(rh_action != NULL) { rh_actions = g_list_append(NULL, rh_action); } else if(rh_action == NULL && rsc != NULL) { rh_actions = find_actions( rsc->actions, order->rh_action_task, NULL); if(rh_actions == NULL) { crm_debug_4("No RH-Side (%s/%s) found for constraint..." " ignoring", rsc->id, order->rh_action_task); crm_debug_4("LH-Side was: (%s/%s)", order->lh_rsc?order->lh_rsc->id:order->lh_action?order->lh_action->rsc->id:"", order->lh_action_task); return; } } else if(rh_action == NULL) { crm_debug_4("No RH-Side (%s) specified for constraint..." " ignoring", order->rh_action_task); crm_debug_4("LH-Side was: (%s/%s)", order->lh_rsc?order->lh_rsc->id:order->lh_action?order->lh_action->rsc->id:"", order->lh_action_task); return; } slist_iter( rh_action_iter, action_t, rh_actions, lpc, if(lh_action) { order_actions(lh_action, rh_action_iter, order->type); } else if(order->type == pe_ordering_restart) { rh_action_iter->runnable = FALSE; } ); pe_free_shallow_adv(rh_actions, FALSE); } void native_rsc_location(resource_t *rsc, rsc_to_node_t *constraint) { GListPtr or_list; crm_debug_2("Applying %s (%s) to %s", constraint->id, role2text(constraint->role_filter), rsc->id); /* take "lifetime" into account */ if(constraint == NULL) { pe_err("Constraint is NULL"); return; } else if(rsc == NULL) { pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id); return; } else if(constraint->role_filter > 0 && constraint->role_filter != rsc->next_role) { crm_debug("Constraint (%s) is not active (role : %s)", constraint->id, role2text(constraint->role_filter)); return; } else if(is_active(constraint) == FALSE) { crm_debug_2("Constraint (%s) is not active", constraint->id); return; } if(constraint->node_list_rh == NULL) { crm_debug_2("RHS of constraint %s is NULL", constraint->id); return; } print_resource(LOG_DEBUG_3, "before update: ", rsc, TRUE); or_list = node_list_or( rsc->allowed_nodes, constraint->node_list_rh, FALSE); pe_free_shallow(rsc->allowed_nodes); rsc->allowed_nodes = or_list; print_resource(LOG_DEBUG_3, "after update: ", rsc, TRUE); } void native_expand(resource_t *rsc, pe_working_set_t *data_set) { slist_iter( action, action_t, rsc->actions, lpc, crm_debug_4("processing action %d for rsc=%s", action->id, rsc->id); graph_element_from_action(action, data_set); ); } void native_rsc_colocation_rh_must(resource_t *rsc_lh, gboolean update_lh, resource_t *rsc_rh, gboolean update_rh) { gboolean do_merge = FALSE; GListPtr old_list = NULL; GListPtr merged_node_list = NULL; int max_pri = rsc_lh->effective_priority; if(max_pri < rsc_rh->effective_priority) { max_pri = rsc_rh->effective_priority; } rsc_lh->effective_priority = max_pri; rsc_rh->effective_priority = max_pri; crm_debug_2("Colocating %s with %s." " Update LHS: %s, Update RHS: %s", rsc_lh->id, rsc_rh->id, update_lh?"true":"false", update_rh?"true":"false"); if(rsc_lh->color && rsc_rh->color) { do_merge = TRUE; merged_node_list = node_list_and( rsc_lh->color->details->candidate_nodes, rsc_rh->color->details->candidate_nodes, TRUE); } else if(rsc_lh->color) { do_merge = TRUE; merged_node_list = node_list_and( rsc_lh->color->details->candidate_nodes, rsc_rh->allowed_nodes, TRUE); } else if(rsc_rh->color) { do_merge = TRUE; merged_node_list = node_list_and( rsc_lh->allowed_nodes, rsc_rh->color->details->candidate_nodes, TRUE); } if(update_lh && rsc_rh != rsc_lh) { CRM_CHECK(rsc_lh->color != rsc_rh->color, return); crm_free(rsc_lh->color); rsc_lh->runnable = rsc_rh->runnable; rsc_lh->provisional = rsc_rh->provisional; CRM_CHECK(rsc_rh->color != NULL, return); native_assign_color(rsc_lh, rsc_rh->color); } if(update_rh && rsc_rh != rsc_lh) { CRM_CHECK(rsc_lh->color != rsc_rh->color, return); crm_free(rsc_rh->color); rsc_rh->runnable = rsc_lh->runnable; rsc_rh->provisional = rsc_lh->provisional; CRM_CHECK(rsc_lh->color != NULL, return); native_assign_color(rsc_rh, rsc_lh->color); } if((update_rh || update_lh) && do_merge) { crm_debug_4("Merging candidate nodes"); old_list = rsc_rh->color->details->candidate_nodes; rsc_rh->color->details->candidate_nodes = merged_node_list; pe_free_shallow(old_list); } crm_debug_4("Finished processing pecs_must constraint"); } void native_rsc_colocation_rh_mustnot(resource_t *rsc_lh, gboolean update_lh, resource_t *rsc_rh, gboolean update_rh) { color_t *color_lh = NULL; color_t *color_rh = NULL; crm_debug_4("Processing pecs_must_not constraint"); /* pecs_must_not */ color_rh = rsc_rh->color; color_lh = rsc_lh->color; if(update_lh) { if(rsc_lh->provisional && color_rh != NULL) { color_lh = add_color(rsc_lh, color_rh); color_lh->local_weight = -INFINITY; crm_debug_2("LH: Removed color %d from resource %s", color_lh->id, rsc_lh->id); crm_action_debug_3( print_color("Removed LH", color_lh, FALSE)); print_resource(LOG_DEBUG_3, "Modified LH", rsc_lh,TRUE); } else if(rsc_lh->provisional) { } else if(color_lh && color_lh->details->pending) { node_t *node_lh = NULL; node_lh = pe_find_node_id( color_lh->details->candidate_nodes, safe_val5(NULL, color_rh, details, chosen_node, details, id)); if(node_lh != NULL) { node_lh->weight = -INFINITY; crm_debug_2("LH: Removed node %s from color %d", node_lh->details->uname, color_lh->id); crm_action_debug_3( print_node("Removed LH", node_lh, FALSE)); crm_action_debug_3( print_color("Modified LH", color_lh, FALSE)); } } else { /* error, rsc marked as unrunnable above */ pe_warn("lh else"); } } /* in case anything was modified */ color_rh = rsc_rh->color; color_lh = rsc_lh->color; if(update_rh) { if(rsc_rh->provisional && color_lh != NULL) { color_rh = add_color(rsc_lh, color_lh); color_rh->local_weight = -INFINITY; crm_debug_2("RH: Removed color %d from resource %s", color_rh->id, rsc_rh->id); crm_action_debug_3( print_color("Removed RH", color_rh, FALSE)); print_resource(LOG_DEBUG_3, "Modified RH", rsc_rh, TRUE); } else if(rsc_rh->provisional) { } else if(color_rh && color_rh->details->pending) { node_t *node_rh = NULL; node_rh = pe_find_node_id( color_rh->details->candidate_nodes, safe_val5(NULL, color_lh, details, chosen_node, details, id)); if(node_rh != NULL) { node_rh->weight = -INFINITY; crm_debug_2("RH: Removed node %s from color %d", node_rh->details->uname, color_rh->id); crm_action_debug_3( print_node("Removed RH", node_rh, FALSE)); crm_action_debug_3( print_color("Modified RH", color_rh, FALSE)); } } else { /* error, rsc marked as unrunnable above */ pe_warn("rh else"); } } } void native_agent_constraints(resource_t *rsc) { } static gint sort_color_weight(gconstpointer a, gconstpointer b) { const color_t *color1 = (const color_t*)a; const color_t *color2 = (const color_t*)b; if(a == NULL) { return 1; } if(b == NULL) { return -1; } if(color1->local_weight > color2->local_weight) { return -1; } if(color1->local_weight < color2->local_weight) { return 1; } return 0; } gboolean native_choose_color(resource_t *rsc, color_t *no_color) { GListPtr sorted_colors = NULL; if(rsc->runnable == FALSE) { native_assign_color(rsc, no_color); } if(rsc->provisional == FALSE) { return !rsc->provisional; } sorted_colors = g_list_sort( rsc->candidate_colors, sort_color_weight); rsc->candidate_colors = sorted_colors; crm_debug_2("Choose a color from %d possibilities", g_list_length(sorted_colors)); slist_iter( this_color, color_t, rsc->candidate_colors, lpc, GListPtr intersection = NULL; GListPtr minus = NULL; int len = 0; if(this_color == NULL) { pe_err("color was NULL"); continue; } else if(this_color->local_weight < 0) { /* no valid color available */ crm_debug("no valid color available"); break; } else if(rsc->effective_priority < this_color->details->highest_priority) { minus = node_list_minus( this_color->details->candidate_nodes, rsc->allowed_nodes, TRUE); len = g_list_length(minus); pe_free_shallow(minus); } else { intersection = node_list_and( this_color->details->candidate_nodes, rsc->allowed_nodes, TRUE); len = g_list_length(intersection); pe_free_shallow(intersection); } if(len > 0) { crm_debug("Assigning color to %s", rsc->id); native_assign_color(rsc, this_color); break; } ); return !rsc->provisional; } void native_assign_color(resource_t *rsc, color_t *color) { color_t *local_color = add_color(rsc, color); GListPtr intersection = NULL; GListPtr old_list = NULL; rsc->provisional = FALSE; CRM_CHECK(local_color != NULL, return); local_color->details->allocated_resources = g_list_append(local_color->details->allocated_resources,rsc); if(rsc->variant == pe_native) { (local_color->details->num_resources)++; rsc->color = copy_color(local_color); crm_debug_3("Created intersection for color %d", local_color->id); intersection = node_list_and( local_color->details->candidate_nodes, rsc->allowed_nodes, FALSE); old_list = local_color->details->candidate_nodes; pe_free_shallow(old_list); local_color->details->candidate_nodes = intersection; } crm_debug_2("Colored resource %s with color %d", rsc->id, local_color->id); return; } void native_update_node_weight(resource_t *rsc, rsc_to_node_t *cons, node_t *cons_node, GListPtr nodes) { node_t *node_rh = NULL; CRM_CHECK(cons_node != NULL, return); node_rh = pe_find_node_id( rsc->allowed_nodes, cons_node->details->id); if(node_rh == NULL) { pe_err("Node not found - adding %s to %s", cons_node->details->id, rsc->id); node_rh = node_copy(cons_node); rsc->allowed_nodes = g_list_append( rsc->allowed_nodes, node_rh); node_rh = pe_find_node_id( rsc->allowed_nodes, cons_node->details->id); CRM_CHECK(node_rh != NULL, return); return; } CRM_CHECK(node_rh != NULL, return); if(node_rh == NULL) { pe_err("Node not found - cant update"); return; } if(node_rh->weight >= INFINITY && cons_node->weight <= -INFINITY) { pe_err("Constraint \"%s\" mixes +/- INFINITY (%s)", cons->id, rsc->id); } else if(node_rh->details->shutdown == TRUE || node_rh->details->online == FALSE || node_rh->details->unclean == TRUE) { } else if(node_rh->weight <= -INFINITY && cons_node->weight >= INFINITY) { pe_err("Constraint \"%s\" mixes +/- INFINITY (%s)", cons->id, rsc->id); } if(node_rh->fixed) { /* warning */ crm_debug_2("Constraint %s is irrelevant as the" " weight of node %s is fixed as %d (%s).", cons->id, node_rh->details->uname, node_rh->weight, rsc->id); return; } crm_debug_3("Constraint %s, node %s, rsc %s: %d + %d", cons->id, node_rh->details->uname, rsc->id, node_rh->weight, cons_node->weight); node_rh->weight = merge_weights(node_rh->weight, cons_node->weight); if(node_rh->weight <= -INFINITY) { crm_debug_3("Constraint %s (-INFINITY): node %s weight %d (%s).", cons->id, node_rh->details->uname, node_rh->weight, rsc->id); } else if(node_rh->weight >= INFINITY) { crm_debug_3("Constraint %s (+INFINITY): node %s weight %d (%s).", cons->id, node_rh->details->uname, node_rh->weight, rsc->id); } else { crm_debug_3("Constraint %s (%d): node %s weight %d (%s).", cons->id, cons_node->weight, node_rh->details->uname, node_rh->weight, rsc->id); } if(node_rh->weight < 0) { node_rh->fixed = TRUE; } crm_action_debug_3(print_node("Updated", node_rh, FALSE)); return; } gboolean native_constraint_violated( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { GListPtr result = NULL; color_t *color_lh = NULL; color_t *color_rh = NULL; GListPtr candidate_nodes_lh = NULL; GListPtr candidate_nodes_rh = NULL; gboolean matched = FALSE; color_lh = rsc_lh->color; color_rh = rsc_rh->color; if(constraint->strength == pecs_must_not) { matched = TRUE; } if(rsc_lh->provisional || rsc_rh->provisional) { return FALSE; } if(color_lh == NULL && color_rh == NULL) { return matched; } else if(color_lh == NULL || color_rh == NULL) { return !matched; } else if(color_lh->details->pending && color_rh->details->pending) { candidate_nodes_lh = color_lh->details->candidate_nodes; candidate_nodes_rh = color_rh->details->candidate_nodes; } else if(color_lh->details->pending == FALSE && color_rh->details->pending == FALSE) { if(color_lh->details->chosen_node == NULL && color_rh->details->chosen_node == NULL) { return matched; } else if(color_lh->details->chosen_node == NULL || color_rh->details->chosen_node == NULL) { return !matched; } else if(safe_str_eq( color_lh->details->chosen_node->details->id, color_rh->details->chosen_node->details->id)) { return matched; } return !matched; } else if(color_lh->details->pending) { candidate_nodes_lh = color_lh->details->candidate_nodes; candidate_nodes_rh = g_list_append( NULL, color_rh->details->chosen_node); } else if(color_rh->details->pending) { candidate_nodes_rh = color_rh->details->candidate_nodes; candidate_nodes_lh = g_list_append( NULL, color_lh->details->chosen_node); } result = node_list_and(candidate_nodes_lh, candidate_nodes_rh, TRUE); if(g_list_length(result) == 0 && constraint->strength == pecs_must) { /* free result */ return TRUE; } return FALSE; } /* * Remove any nodes with a -ve weight */ void filter_nodes(resource_t *rsc) { print_resource(LOG_DEBUG_3, "Filtering nodes for: ", rsc, FALSE); slist_iter( node, node_t, rsc->allowed_nodes, lpc, if(node == NULL) { pe_err("Invalid NULL node"); } else if(node->weight < 0.0 || node->details->shutdown || node->details->online == FALSE || node->details->type == node_ping) { crm_action_debug_3(print_node("Removing", node, FALSE)); rsc->allowed_nodes = g_list_remove(rsc->allowed_nodes, node); crm_free(node); lpc = -1; /* restart the loop */ } ); } void create_notifications(resource_t *rsc, pe_working_set_t *data_set) { if(rsc->notify == FALSE) { return; } /* slist_iter( */ /* action, action_t, rsc->actions, lpc, */ /* ); */ } static void register_activity(resource_t *rsc, enum action_tasks task, node_t *node, notify_data_t *n_data) { notify_entry_t *entry = NULL; crm_malloc0(entry, sizeof(notify_entry_t)); entry->rsc = rsc; entry->node = node; switch(task) { case start_rsc: n_data->start = g_list_append(n_data->start, entry); break; case stop_rsc: n_data->stop = g_list_append(n_data->stop, entry); break; case action_promote: n_data->promote = g_list_append(n_data->promote, entry); break; case action_demote: n_data->demote = g_list_append(n_data->demote, entry); break; default: crm_err("Unsupported notify action: %s", task2text(task)); break; } } static void register_state(resource_t *rsc, node_t *on_node, notify_data_t *n_data) { notify_entry_t *entry = NULL; crm_malloc0(entry, sizeof(notify_entry_t)); entry->rsc = rsc; entry->node = on_node; crm_debug_2("%s state: %s", rsc->id, role2text(rsc->next_role)); switch(rsc->next_role) { case RSC_ROLE_STOPPED: /* n_data->inactive = g_list_append(n_data->inactive, entry); */ crm_free(entry); break; case RSC_ROLE_STARTED: n_data->active = g_list_append(n_data->active, entry); break; case RSC_ROLE_SLAVE: n_data->slave = g_list_append(n_data->slave, entry); break; case RSC_ROLE_MASTER: n_data->master = g_list_append(n_data->master, entry); break; default: crm_err("Unsupported notify role"); break; } } void native_create_notify_element(resource_t *rsc, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set) { node_t *next_node = NULL; gboolean registered = FALSE; char *op_key = NULL; GListPtr possible_matches = NULL; enum action_tasks task = text2task(op->task); if(op->pre_notify == NULL || op->post_notify == NULL) { /* no notifications required */ crm_debug_4("No notificaitons required for %s", op->task); return; } if(rsc->color != NULL) { next_node = rsc->color->details->chosen_node; } op_key = generate_op_key(rsc->id, op->task, 0); possible_matches = find_actions(rsc->actions, op_key, NULL); crm_debug_2("Creating notificaitons for: %s (%s->%s)", op->uuid, role2text(rsc->role), role2text(rsc->next_role)); if(rsc->role == rsc->next_role) { register_state(rsc, next_node, n_data); } slist_iter( local_op, action_t, possible_matches, lpc, local_op->notify_keys = n_data->keys; if(local_op->optional == FALSE) { registered = TRUE; register_activity(rsc, task, local_op->node, n_data); } ); /* stop / demote */ if(rsc->role != RSC_ROLE_STOPPED) { if(task == stop_rsc || task == action_demote) { slist_iter( current_node, node_t, rsc->running_on, lpc, pe_pre_notify(rsc, current_node, op, n_data, data_set); if(task == action_demote || registered == FALSE) { pe_post_notify(rsc, current_node, op, n_data, data_set); } ); } } /* start / promote */ if(rsc->next_role != RSC_ROLE_STOPPED) { CRM_CHECK(next_node != NULL,;); if(next_node == NULL) { pe_proc_err("next role: %s", role2text(rsc->next_role)); } else if(task == start_rsc || task == action_promote) { if(task != start_rsc || registered == FALSE) { pe_pre_notify(rsc, next_node, op, n_data, data_set); } pe_post_notify(rsc, next_node, op, n_data, data_set); } } crm_free(op_key); pe_free_shallow_adv(possible_matches, FALSE); } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { char *meta_key = crm_concat(CRM_META, key, '_'); g_hash_table_replace(user_data, meta_key, crm_strdup(value)); } static action_t * pe_notify(resource_t *rsc, node_t *node, action_t *op, action_t *confirm, notify_data_t *n_data, pe_working_set_t *data_set) { char *key = NULL; action_t *trigger = NULL; const char *value = NULL; const char *task = NULL; if(op == NULL || confirm == NULL) { crm_debug_2("Op=%p confirm=%p", op, confirm); return NULL; } CRM_CHECK(node != NULL, return NULL); if(node->details->online == FALSE) { crm_info("Skipping notification for %s", rsc->id); return NULL; } value = g_hash_table_lookup(op->meta, "notify_type"); task = g_hash_table_lookup(op->meta, "notify_operation"); crm_debug_2("Creating actions for %s: %s (%s-%s)", op->uuid, rsc->id, value, task); key = generate_notify_key(rsc->id, value, task); trigger = custom_action(rsc, key, op->task, node, op->optional, TRUE, data_set); g_hash_table_foreach(op->meta, dup_attr, trigger->extra); trigger->notify_keys = n_data->keys; /* pseudo_notify before notify */ crm_debug_3("Ordering %s before %s (%d->%d)", op->uuid, trigger->uuid, trigger->id, op->id); order_actions(op, trigger, pe_ordering_manditory); value = g_hash_table_lookup(op->meta, "notify_confirm"); if(crm_is_true(value)) { /* notify before pseudo_notified */ crm_debug_3("Ordering %s before %s (%d->%d)", trigger->uuid, confirm->uuid, confirm->id, trigger->id); order_actions(trigger, confirm, pe_ordering_manditory); } return trigger; } void pe_pre_notify(resource_t *rsc, node_t *node, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set) { crm_debug_2("%s: %s", rsc->id, op->uuid); pe_notify(rsc, node, op->pre_notify, op->pre_notified, n_data, data_set); } void pe_post_notify(resource_t *rsc, node_t *node, action_t *op, notify_data_t *n_data, pe_working_set_t *data_set) { action_t *notify = NULL; CRM_CHECK(op != NULL, return); CRM_CHECK(rsc != NULL, return); crm_debug_2("%s: %s", rsc->id, op->uuid); notify = pe_notify(rsc, node, op->post_notify, op->post_notified, n_data, data_set); if(notify != NULL) { notify->priority = INFINITY; } notify = op->post_notified; if(notify != NULL) { notify->priority = INFINITY; slist_iter( mon, action_t, rsc->actions, lpc, const char *interval = g_hash_table_lookup(mon->meta, "interval"); if(interval == NULL || safe_str_eq(interval, "0")) { crm_debug_3("Skipping %s: interval", mon->uuid); continue; } else if(safe_str_eq(mon->task, "cancel")) { crm_debug_3("Skipping %s: cancel", mon->uuid); continue; } order_actions(notify, mon, pe_ordering_optional); ); } } void NoRoleChange(resource_t *rsc, node_t *current, node_t *next, pe_working_set_t *data_set) { action_t *start = NULL; action_t *stop = NULL; GListPtr possible_matches = NULL; crm_debug("Executing: %s (role=%s)",rsc->id, role2text(rsc->next_role)); if(current == NULL || next == NULL) { return; } /* use StartRsc/StopRsc */ if(safe_str_neq(current->details->id, next->details->id)) { crm_notice("Move resource %s\t(%s -> %s)", rsc->id, current->details->uname, next->details->uname); stop = stop_action(rsc, current, FALSE); start = start_action(rsc, next, FALSE); possible_matches = find_recurring_actions(rsc->actions, next); slist_iter(match, action_t, possible_matches, lpc, if(match->optional == FALSE) { crm_err("Found bad recurring action: %s", match->uuid); match->optional = TRUE; } ); if(data_set->remove_after_stop) { DeleteRsc(rsc, current, data_set); } } else { if(rsc->failed) { crm_notice("Recover resource %s\t(%s)", rsc->id, next->details->uname); stop = stop_action(rsc, current, FALSE); start = start_action(rsc, next, FALSE); /* /\* make the restart required *\/ */ /* order_stop_start(rsc, rsc, pe_ordering_manditory); */ } else if(rsc->start_pending) { start = start_action(rsc, next, TRUE); if(start->runnable) { /* wait for StartRsc() to be called */ rsc->role = RSC_ROLE_STOPPED; } else { /* wait for StopRsc() to be called */ rsc->next_role = RSC_ROLE_STOPPED; } } else { stop = stop_action(rsc, current, TRUE); start = start_action(rsc, next, TRUE); stop->optional = start->optional; if(start->runnable == FALSE) { rsc->next_role = RSC_ROLE_STOPPED; } else if(start->optional) { crm_notice("Leave resource %s\t(%s)", rsc->id, next->details->uname); } else { crm_notice("Restart resource %s\t(%s)", rsc->id, next->details->uname); } } } } gboolean StopRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { action_t *stop = NULL; crm_debug_2("Executing: %s", rsc->id); slist_iter( current, node_t, rsc->running_on, lpc, crm_notice(" %s\tStop %s", current->details->uname, rsc->id); stop = stop_action(rsc, current, FALSE); if(data_set->remove_after_stop) { DeleteRsc(rsc, current, data_set); } ); return TRUE; } gboolean StartRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { action_t *start = NULL; crm_debug_2("Executing: %s", rsc->id); start = start_action(rsc, next, TRUE); if(start->runnable) { crm_notice(" %s\tStart %s", next->details->uname, rsc->id); start->optional = FALSE; } return TRUE; } gboolean PromoteRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { char *key = NULL; gboolean runnable = TRUE; GListPtr action_list = NULL; crm_debug_2("Executing: %s", rsc->id); CRM_CHECK(rsc->next_role == RSC_ROLE_MASTER, return FALSE); key = start_key(rsc); action_list = find_actions_exact(rsc->actions, key, next); crm_free(key); slist_iter(start, action_t, action_list, lpc, if(start->runnable == FALSE) { runnable = FALSE; } ); if(runnable) { promote_action(rsc, next, FALSE); crm_notice("%s\tPromote %s", next->details->uname, rsc->id); return TRUE; } crm_debug("%s\tPromote %s (canceled)", next->details->uname, rsc->id); key = promote_key(rsc); action_list = find_actions_exact(rsc->actions, key, next); crm_free(key); slist_iter(promote, action_t, action_list, lpc, promote->runnable = FALSE; ); return TRUE; } gboolean DemoteRsc(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { crm_debug_2("Executing: %s", rsc->id); /* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */ slist_iter( current, node_t, rsc->running_on, lpc, crm_notice("%s\tDeomote %s", current->details->uname, rsc->id); demote_action(rsc, current, FALSE); ); return TRUE; } gboolean RoleError(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { crm_debug("Executing: %s", rsc->id); CRM_CHECK(FALSE, return FALSE); return FALSE; } gboolean NullOp(resource_t *rsc, node_t *next, pe_working_set_t *data_set) { crm_debug("Executing: %s", rsc->id); return FALSE; } gboolean native_create_probe(resource_t *rsc, node_t *node, action_t *complete, gboolean force, pe_working_set_t *data_set) { char *key = NULL; char *target_rc = NULL; action_t *probe = NULL; node_t *running = NULL; CRM_CHECK(node != NULL, return FALSE); if(rsc->orphan) { crm_debug_2("Skipping orphan: %s", rsc->id); return FALSE; } running = pe_find_node_id(rsc->known_on, node->details->id); if(force == FALSE && running != NULL) { /* we already know the status of the resource on this node */ crm_debug_3("Skipping active: %s", rsc->id); return FALSE; } key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0); probe = custom_action(rsc, key, CRMD_ACTION_STATUS, node, FALSE, TRUE, data_set); probe->priority = INFINITY; running = pe_find_node_id(rsc->running_on, node->details->id); if(running == NULL) { target_rc = crm_itoa(EXECRA_NOT_RUNNING); add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, target_rc); crm_free(target_rc); } crm_notice("%s: Created probe for %s", node->details->uname, rsc->id); custom_action_order(rsc, NULL, probe, rsc, NULL, complete, pe_ordering_manditory, data_set); return TRUE; } void native_stonith_ordering( resource_t *rsc, action_t *stonith_op, pe_working_set_t *data_set) { gboolean is_unprotected = FALSE; gboolean run_unprotected = TRUE; const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS); if(stonith_op != NULL && safe_str_eq(class, "stonith")) { char *key = start_key(rsc); crm_debug("Ordering %s before stonith op", key); custom_action_order( rsc, key, NULL, NULL, crm_strdup(CRM_OP_FENCE), stonith_op, pe_ordering_optional, data_set); return; } slist_iter(action, action_t, rsc->actions, lpc2, if(action->needs != rsc_req_stonith) { continue; } if(stonith_op != NULL) { custom_action_order( NULL, crm_strdup(CRM_OP_FENCE), stonith_op, rsc, NULL, action, pe_ordering_manditory, data_set); } else if(run_unprotected == FALSE) { /* mark the start unrunnable */ action->runnable = FALSE; } else { is_unprotected = TRUE; } ); if(is_unprotected) { pe_err("SHARED RESOURCE %s IS NOT PROTECTED:" " Stonith disabled", rsc->id); } } diff --git a/crm/pengine/regression.sh b/crm/pengine/regression.sh index b6d655671c..fb683921ea 100755 --- a/crm/pengine/regression.sh +++ b/crm/pengine/regression.sh @@ -1,218 +1,219 @@ #!/bin/bash # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # . regression.core.sh create_mode="true" echo Generating test outputs for these tests... echo "" echo Done. echo "" echo Performing the following tests... create_mode="false" echo "" do_test simple1 "Offline " do_test simple2 "Start " do_test simple3 "Start 2 " do_test simple4 "Start Failed" do_test simple6 "Stop Start " do_test simple7 "Shutdown " #do_test simple8 "Stonith " #do_test simple9 "Lower version" #do_test simple10 "Higher version" do_test simple11 "Priority (ne)" do_test simple12 "Priority (eq)" do_test simple8 "Stickiness" echo "" do_test params-0 "Params: No change" do_test params-1 "Params: Changed" do_test params-2 "Params: Resource definition" echo "" do_test orphan-0 "Orphan ignore" do_test orphan-1 "Orphan stop" echo "" do_test target-0 "Target Role : baseline" do_test target-1 "Target Role : test" echo "" do_test date-1 "Dates" -d "2005-020" do_test probe-0 "Probe (anon clone)" do_test standby "Standby" do_test comments "Comments" echo "" do_test master-0 "Stopped -> Slave" do_test master-1 "Stopped -> Promote" do_test master-2 "Stopped -> Promote : notify" do_test master-3 "Stopped -> Promote : master location" do_test master-4 "Started -> Promote : master location" do_test master-5 "Promoted -> Promoted" do_test master-6 "Promoted -> Promoted (2)" do_test master-7 "Promoted -> Fenced" do_test master-8 "Promoted -> Fenced -> Moved" do_test master-9 "Stopped + Promotable + No quorum" do_test master-10 "Stopped -> Promotable : notify with monitor" echo "" do_test rsc_dep1 "Must not " do_test rsc_dep3 "Must " do_test rsc_dep5 "Must not 3 " do_test rsc_dep7 "Must 3 " do_test rsc_dep10 "Must (but cant)" do_test rsc_dep2 "Must (running) " do_test rsc_dep8 "Must (running : alt) " do_test rsc_dep4 "Must (running + move)" echo "" do_test order1 "Order start 1 " do_test order2 "Order start 2 " do_test order3 "Order stop " do_test order4 "Order (multiple) " do_test order5 "Order (move) " do_test order6 "Order (move w/ restart) " do_test order7 "Order (manditory) " #echo "" #do_test agent1 "version: lt (empty)" #do_test agent2 "version: eq " #do_test agent3 "version: gt " echo "" do_test attrs1 "string: eq (and) " do_test attrs2 "string: lt / gt (and)" do_test attrs3 "string: ne (or) " do_test attrs4 "string: exists " do_test attrs5 "string: not_exists " do_test attrs6 "is_dc: true " do_test attrs7 "is_dc: false " do_test attrs8 "score_attribute " echo "" do_test mon-rsc-1 "Schedule Monitor - start" do_test mon-rsc-2 "Schedule Monitor - move " do_test mon-rsc-3 "Schedule Monitor - pending start " do_test mon-rsc-4 "Schedule Monitor - move/pending start" echo "" do_test rec-rsc-0 "Resource Recover - no start " do_test rec-rsc-1 "Resource Recover - start " do_test rec-rsc-2 "Resource Recover - monitor " do_test rec-rsc-3 "Resource Recover - stop - ignore" do_test rec-rsc-4 "Resource Recover - stop - block " do_test rec-rsc-5 "Resource Recover - stop - fence " do_test rec-rsc-6 "Resource Recover - multiple - restart" do_test rec-rsc-7 "Resource Recover - multiple - stop " do_test rec-rsc-8 "Resource Recover - multiple - block " echo "" do_test quorum-1 "No quorum - ignore" do_test quorum-2 "No quorum - freeze" do_test quorum-3 "No quorum - stop " do_test quorum-4 "No quorum - start anyway" do_test quorum-5 "No quorum - start anyway (group)" do_test quorum-6 "No quorum - start anyway (clone)" echo "" do_test rec-node-1 "Node Recover - Startup - no fence" do_test rec-node-2 "Node Recover - Startup - fence " do_test rec-node-3 "Node Recover - HA down - no fence" do_test rec-node-4 "Node Recover - HA down - fence " do_test rec-node-5 "Node Recover - CRM down - no fence" do_test rec-node-6 "Node Recover - CRM down - fence " do_test rec-node-7 "Node Recover - no quorum - ignore " do_test rec-node-8 "Node Recover - no quorum - freeze " do_test rec-node-9 "Node Recover - no quorum - stop " do_test rec-node-10 "Node Recover - no quorum - stop w/fence" do_test rec-node-11 "Node Recover - CRM down w/ group - fence " do_test rec-node-12 "Node Recover - nothing active - fence " do_test rec-node-13 "Node Recover - failed resource + shutdown - fence " do_test rec-node-14 "Serialize all stonith's" echo "" do_test multi1 "Multiple Active (stop/start)" #echo "" #do_test complex1 "Complex " echo "" do_test group1 "Group " do_test group2 "Group + Native " do_test group3 "Group + Group " do_test group4 "Group + Native (nothing)" do_test group5 "Group + Native (move) " do_test group6 "Group + Group (move) " do_test group7 "Group colocation" +do_test group13 "Group colocation (cant run)" do_test group8 "Group anti-colocation" do_test group9 "Group recovery" do_test group10 "Group partial recovery" do_test group11 "Group target_role" echo "" do_test inc0 "Incarnation start " do_test inc1 "Incarnation start order " do_test inc2 "Incarnation silent restart, stop, move " do_test inc3 "Inter-incarnation ordering, silent restart, stop, move" do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)" do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)" do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)" do_test inc7 "Clone colocation" do_test inc8 "Clone anti-colocation" do_test inc9 "Non-unique clone" do_test inc10 "Non-unique clone (stop)" echo "" do_test managed-0 "Managed (reference)" do_test managed-1 "Not managed - down " do_test managed-2 "Not managed - up " echo "" do_test interleave-0 "Interleave (reference)" do_test interleave-1 "coloc - not interleaved" do_test interleave-2 "coloc - interleaved " do_test interleave-3 "coloc - interleaved (2)" echo "" do_test notify-0 "Notify reference" do_test notify-1 "Notify simple" do_test notify-2 "Notify simple, confirm" do_test notify-3 "Notify move, confirm" #do_test notify-2 "Notify - 764" echo "" do_test 594 "Bugzilla 594" do_test 662 "Bugzilla 662" do_test 696 "Bugzilla 696" do_test 726 "Bugzilla 726" do_test 735 "Bugzilla 735" do_test 764 "Bugzilla 764" do_test 797 "Bugzilla 797" do_test 829 "Bugzilla 829" do_test 994 "Bugzilla 994" do_test unrunnable-1 "Unrunnable" echo "" test_results diff --git a/crm/pengine/testcases/group13.dot b/crm/pengine/testcases/group13.dot new file mode 100644 index 0000000000..60a891be7b --- /dev/null +++ b/crm/pengine/testcases/group13.dot @@ -0,0 +1,25 @@ +digraph "g" { + size = "30,30" +"probe_complete" [ style="dashed" color="blue" fontcolor="orange" ] +"probe_complete jamesltc" [ style="dashed" color="blue" fontcolor="black" ] +"resource_nfs_stop_0 jamesltc" [ style=bold color="green" fontcolor="black" ] +"nfs_start_0" [ style="dashed" color="blue" fontcolor="orange" ] +"nfs_running_0" [ style="dashed" color="blue" fontcolor="orange" ] +"nfs_stop_0" [ style=bold color="green" fontcolor="orange" ] +"nfs_stopped_0" [ style=bold color="green" fontcolor="orange" ] +"fs_start_0" [ style="dashed" color="blue" fontcolor="orange" ] +"fs_running_0" [ style="dashed" color="blue" fontcolor="orange" ] +"fs_stop_0" [ style="dashed" color="blue" fontcolor="orange" ] +"fs_stopped_0" [ style="dashed" color="blue" fontcolor="orange" ] +"probe_complete jamesltc" -> "probe_complete" [ style = dashed] +"nfs_stop_0" -> "resource_nfs_stop_0 jamesltc" [ style = bold] +"nfs_stop_0" -> "nfs_start_0" [ style = dashed] +"nfs_stopped_0" -> "nfs_start_0" [ style = dashed] +"nfs_start_0" -> "nfs_running_0" [ style = dashed] +"resource_nfs_stop_0 jamesltc" -> "nfs_stopped_0" [ style = bold] +"nfs_stop_0" -> "nfs_stopped_0" [ style = bold] +"fs_stop_0" -> "fs_start_0" [ style = dashed] +"fs_stopped_0" -> "fs_start_0" [ style = dashed] +"fs_start_0" -> "fs_running_0" [ style = dashed] +"fs_stop_0" -> "fs_stopped_0" [ style = dashed] +} diff --git a/crm/pengine/testcases/group13.exp b/crm/pengine/testcases/group13.exp new file mode 100644 index 0000000000..4c2c81a704 --- /dev/null +++ b/crm/pengine/testcases/group13.exp @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crm/pengine/testcases/group13.xml b/crm/pengine/testcases/group13.xml new file mode 100644 index 0000000000..33e35118f7 --- /dev/null +++ b/crm/pengine/testcases/group13.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +