diff --git a/crm/pengine/clone.c b/crm/pengine/clone.c
index 9abf4d1ed9..ffccbb9a30 100644
--- a/crm/pengine/clone.c
+++ b/crm/pengine/clone.c
@@ -1,1271 +1,1283 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <lha_internal.h>
#include <crm/msg_xml.h>
#include <allocate.h>
#include <utils.h>
#include <lib/crm/pengine/utils.h>
#define VARIANT_CLONE 1
#include <lib/crm/pengine/variant.h>
gint sort_clone_instance(gconstpointer a, gconstpointer b);
+resource_t *find_compatible_child(
+ resource_t *local_child, resource_t *rsc, enum rsc_role_e filter);
void clone_create_notifications(
resource_t *rsc, action_t *action, action_t *action_complete,
pe_working_set_t *data_set);
void child_stopping_constraints(
clone_variant_data_t *clone_data,
resource_t *self, resource_t *child, resource_t *last,
pe_working_set_t *data_set);
void child_starting_constraints(
clone_variant_data_t *clone_data,
resource_t *self, resource_t *child, resource_t *last,
pe_working_set_t *data_set);
static node_t *
parent_node_instance(const resource_t *rsc, node_t *node)
{
node_t *ret = NULL;
if(node != NULL) {
ret = pe_find_node_id(
rsc->parent->allowed_nodes, node->details->id);
}
return ret;
}
gint sort_clone_instance(gconstpointer a, gconstpointer b)
{
int level = LOG_DEBUG_3;
node_t *node1 = NULL;
node_t *node2 = NULL;
gboolean can1 = TRUE;
gboolean can2 = TRUE;
const resource_t *resource1 = (const resource_t*)a;
const resource_t *resource2 = (const resource_t*)b;
CRM_ASSERT(resource1 != NULL);
CRM_ASSERT(resource2 != NULL);
/* allocation order:
* - active instances
* - instances running on nodes with the least copies
* - active instances on nodes that cant support them or are to be fenced
* - failed instances
* - inactive instances
*/
do_crm_log(level+1, "%s ? %s", resource1->id, resource2->id);
if(resource1->running_on && resource2->running_on) {
if(g_list_length(resource1->running_on) < g_list_length(resource2->running_on)) {
do_crm_log(level, "%s < %s: running_on", resource1->id, resource2->id);
return -1;
} else if(g_list_length(resource1->running_on) > g_list_length(resource2->running_on)) {
do_crm_log(level, "%s > %s: running_on", resource1->id, resource2->id);
return 1;
}
}
if(resource1->running_on) {
node1 = resource1->running_on->data;
}
if(resource2->running_on) {
node2 = resource2->running_on->data;
}
if(resource1->priority < resource2->priority) {
do_crm_log(level, "%s < %s: score", resource1->id, resource2->id);
return 1;
} else if(resource1->priority > resource2->priority) {
do_crm_log(level, "%s > %s: score", resource1->id, resource2->id);
return -1;
}
if(node1 == NULL && node2 == NULL) {
do_crm_log(level, "%s == %s: not active",
resource1->id, resource2->id);
return 0;
}
if(node1 != node2) {
if(node1 == NULL) {
do_crm_log(level, "%s > %s: active", resource1->id, resource2->id);
return 1;
} else if(node2 == NULL) {
do_crm_log(level, "%s < %s: active", resource1->id, resource2->id);
return -1;
}
}
can1 = can_run_resources(node1);
can2 = can_run_resources(node2);
if(can1 != can2) {
if(can1) {
do_crm_log(level, "%s < %s: can", resource1->id, resource2->id);
return -1;
}
do_crm_log(level, "%s > %s: can", resource1->id, resource2->id);
return 1;
}
node1 = parent_node_instance(resource1, node1);
node2 = parent_node_instance(resource2, node2);
if(node1 != NULL && node2 == NULL) {
do_crm_log(level, "%s < %s: not allowed", resource1->id, resource2->id);
return -1;
} else if(node1 == NULL && node2 != NULL) {
do_crm_log(level, "%s > %s: not allowed", resource1->id, resource2->id);
return 1;
}
if(node1 == NULL) {
do_crm_log(level, "%s == %s: not allowed", resource1->id, resource2->id);
return 0;
}
if(node1->count < node2->count) {
do_crm_log(level, "%s < %s: count", resource1->id, resource2->id);
return -1;
} else if(node1->count > node2->count) {
do_crm_log(level, "%s > %s: count", resource1->id, resource2->id);
return 1;
}
if(node1->weight < node2->weight) {
do_crm_log(level, "%s < %s: node score", resource1->id, resource2->id);
return 1;
} else if(node1->weight > node2->weight) {
do_crm_log(level, "%s > %s: node score", resource1->id, resource2->id);
return -1;
}
do_crm_log(level, "%s == %s: default %d", resource1->id, resource2->id, node2->weight);
return 0;
}
static node_t *
can_run_instance(resource_t *rsc, node_t *node)
{
node_t *local_node = NULL;
clone_variant_data_t *clone_data = NULL;
if(can_run_resources(node) == FALSE) {
goto bail;
}
local_node = parent_node_instance(rsc, node);
get_clone_variant_data(clone_data, rsc->parent);
if(local_node == NULL) {
crm_warn("%s cannot run on %s: node not allowed",
rsc->id, node->details->uname);
goto bail;
} else if(local_node->count < clone_data->clone_node_max) {
return local_node;
} else {
crm_debug_2("%s cannot run on %s: node full",
rsc->id, node->details->uname);
}
bail:
if(node) {
common_update_score(rsc, node->details->id, -INFINITY);
}
return NULL;
}
static node_t *
color_instance(resource_t *rsc, pe_working_set_t *data_set)
{
node_t *local_node = NULL;
node_t *chosen = NULL;
crm_debug_2("Processing %s", rsc->id);
if(is_not_set(rsc->flags, pe_rsc_provisional)) {
return rsc->allocated_to;
} else if(is_set(rsc->flags, pe_rsc_allocating)) {
crm_debug("Dependancy loop detected involving %s", rsc->id);
return NULL;
}
if(rsc->allowed_nodes) {
slist_iter(try_node, node_t, rsc->allowed_nodes, lpc,
can_run_instance(rsc, try_node);
);
}
chosen = rsc->cmds->color(rsc, data_set);
if(chosen) {
local_node = pe_find_node_id(
rsc->parent->allowed_nodes, chosen->details->id);
if(local_node) {
local_node->count++;
} else if(is_set(rsc->flags, pe_rsc_managed)) {
/* what to do? we can't enforce per-node limits in this case */
crm_config_err("%s not found in %s (list=%d)",
chosen->details->id, rsc->parent->id,
g_list_length(rsc->parent->allowed_nodes));
}
}
return chosen;
}
static void append_parent_colocation(resource_t *rsc, resource_t *child)
{
slist_iter(cons, rsc_colocation_t, rsc->rsc_cons, lpc,
child->rsc_cons = g_list_append(child->rsc_cons, cons));
slist_iter(cons, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
child->rsc_cons_lhs = g_list_append(child->rsc_cons_lhs, cons));
}
node_t *
clone_color(resource_t *rsc, pe_working_set_t *data_set)
{
int allocated = 0;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if(is_not_set(rsc->flags, pe_rsc_provisional)) {
return NULL;
} else if(is_set(rsc->flags, pe_rsc_allocating)) {
crm_debug("Dependancy loop detected involving %s", rsc->id);
return NULL;
}
set_bit(rsc->flags, pe_rsc_allocating);
crm_debug_2("Processing %s", rsc->id);
/* this information is used by sort_clone_instance() when deciding in which
* order to allocate clone instances
*/
slist_iter(
constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
constraint->score/INFINITY, TRUE);
);
dump_node_scores(LOG_DEBUG_2, rsc, __FUNCTION__, rsc->allowed_nodes);
/* count now tracks the number of clones currently allocated */
slist_iter(node, node_t, rsc->allowed_nodes, lpc,
node->count = 0;
);
slist_iter(child, resource_t, rsc->children, lpc,
if(g_list_length(child->running_on) > 0) {
node_t *child_node = child->running_on->data;
node_t *local_node = parent_node_instance(
child, child->running_on->data);
if(local_node) {
local_node->count++;
} else {
crm_err("%s is running on %s which isn't allowed",
child->id, child_node->details->uname);
}
}
);
rsc->children = g_list_sort(rsc->children, sort_clone_instance);
/* count now tracks the number of clones we have allocated */
slist_iter(node, node_t, rsc->allowed_nodes, lpc,
node->count = 0;
);
rsc->allowed_nodes = g_list_sort(
rsc->allowed_nodes, sort_node_weight);
slist_iter(child, resource_t, rsc->children, lpc,
if(allocated >= clone_data->clone_max) {
crm_debug("Child %s not allocated - limit reached", child->id);
resource_location(child, NULL, -INFINITY, "clone_color:limit_reached", data_set);
} else {
append_parent_colocation(rsc, child);
}
if(color_instance(child, data_set)) {
allocated++;
}
);
crm_debug("Allocated %d %s instances of a possible %d",
allocated, rsc->id, clone_data->clone_max);
clear_bit(rsc->flags, pe_rsc_provisional);
clear_bit(rsc->flags, pe_rsc_allocating);
return NULL;
}
static void
clone_update_pseudo_status(
resource_t *child, gboolean *stopping, gboolean *starting)
{
CRM_ASSERT(stopping != NULL);
CRM_ASSERT(starting != NULL);
slist_iter(
action, action_t, child->actions, lpc,
if(*starting && *stopping) {
return;
} else if(action->optional) {
crm_debug_3("Skipping optional: %s", action->uuid);
continue;
} else if(action->pseudo == FALSE && action->runnable == FALSE){
crm_debug_3("Skipping unrunnable: %s", action->uuid);
continue;
} else if(safe_str_eq(CRMD_ACTION_STOP, action->task)) {
crm_debug_2("Stopping due to: %s", action->uuid);
*stopping = TRUE;
} else if(safe_str_eq(CRMD_ACTION_START, action->task)) {
if(action->runnable == FALSE) {
crm_debug_3("Skipping pseudo-op: %s run=%d, pseudo=%d",
action->uuid, action->runnable, action->pseudo);
} else {
crm_debug_2("Starting due to: %s", action->uuid);
crm_debug_3("%s run=%d, pseudo=%d",
action->uuid, action->runnable, action->pseudo);
*starting = TRUE;
}
}
);
}
void clone_create_actions(resource_t *rsc, pe_working_set_t *data_set)
{
gboolean child_starting = FALSE;
gboolean child_stopping = FALSE;
action_t *stop = NULL;
action_t *start = NULL;
action_t *action_complete = NULL;
resource_t *last_start_rsc = NULL;
resource_t *last_stop_rsc = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_debug_2("Creating actions for %s", rsc->id);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->cmds->create_actions(child_rsc, data_set);
clone_update_pseudo_status(
child_rsc, &child_stopping, &child_starting);
if(is_set(child_rsc->flags, pe_rsc_starting)) {
last_start_rsc = child_rsc;
}
if(is_set(child_rsc->flags, pe_rsc_stopping)) {
last_stop_rsc = child_rsc;
}
);
/* start */
start = start_action(rsc, NULL, !child_starting);
action_complete = custom_action(
rsc, started_key(rsc),
CRMD_ACTION_STARTED, NULL, !child_starting, TRUE, data_set);
start->pseudo = TRUE;
start->runnable = TRUE;
action_complete->pseudo = TRUE;
action_complete->runnable = TRUE;
action_complete->priority = INFINITY;
/* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */
child_starting_constraints(clone_data, rsc, NULL, last_start_rsc, data_set);
clone_create_notifications(
rsc, start, action_complete, data_set);
/* stop */
stop = stop_action(rsc, NULL, !child_stopping);
action_complete = custom_action(
rsc, stopped_key(rsc),
CRMD_ACTION_STOPPED, NULL, !child_stopping, TRUE, data_set);
stop->pseudo = TRUE;
stop->runnable = TRUE;
action_complete->pseudo = TRUE;
action_complete->runnable = TRUE;
action_complete->priority = INFINITY;
/* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */
child_stopping_constraints(clone_data, rsc, NULL, last_stop_rsc, data_set);
clone_create_notifications(rsc, stop, action_complete, data_set);
rsc->actions = rsc->actions;
if(stop->post_notified != NULL && start->pre_notify != NULL) {
order_actions(stop->post_notified, start->pre_notify, pe_order_optional);
}
}
void
clone_create_notifications(
resource_t *rsc, action_t *action, action_t *action_complete,
pe_working_set_t *data_set)
{
/*
* pre_notify -> pre_notify_complete -> pseudo_action
* -> (real actions) -> pseudo_action_complete
* -> post_notify -> post_notify_complete
*
* if the pre_noitfy requires confirmation,
* then a list of confirmations will be added as triggers
* to pseudo_action in clone_expand()
*/
action_t *notify = NULL;
action_t *notify_complete = NULL;
enum action_tasks task;
char *notify_key = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if(is_not_set(rsc->flags, pe_rsc_notify)) {
return;
}
task = text2task(action->task);
/* create pre_notify */
notify_key = generate_notify_key(
rsc->id, "pre", action->task);
notify = custom_action(rsc, notify_key,
CRMD_ACTION_NOTIFY, NULL,
action->optional, TRUE, data_set);
add_hash_param(notify->meta, "notify_type", "pre");
add_hash_param(notify->meta, "notify_operation", action->task);
if(clone_data->notify_confirm) {
add_hash_param(notify->meta, "notify_confirm", "yes");
} else {
add_hash_param(notify->meta, "notify_confirm", "no");
}
/* create pre_notify_complete */
notify_key = generate_notify_key(
rsc->id, "confirmed-pre", action->task);
notify_complete = custom_action(rsc, notify_key,
CRMD_ACTION_NOTIFIED, NULL,
action->optional, TRUE, data_set);
add_hash_param(notify_complete->meta, "notify_type", "pre");
add_hash_param(notify_complete->meta, "notify_operation", action->task);
if(clone_data->notify_confirm) {
add_hash_param(notify->meta, "notify_confirm", "yes");
} else {
add_hash_param(notify->meta, "notify_confirm", "no");
}
notify->pseudo = TRUE;
notify->runnable = TRUE;
notify_complete->pseudo = TRUE;
notify_complete->runnable = TRUE;
/* pre_notify before pre_notify_complete */
custom_action_order(
rsc, NULL, notify,
rsc, NULL, notify_complete,
pe_order_implies_left, data_set);
/* pre_notify_complete before action */
custom_action_order(
rsc, NULL, notify_complete,
rsc, NULL, action,
pe_order_implies_left, data_set);
action->pre_notify = notify;
action->pre_notified = notify_complete;
/* create post_notify */
notify_key = generate_notify_key
(rsc->id, "post", action->task);
notify = custom_action(rsc, notify_key,
CRMD_ACTION_NOTIFY, NULL,
action_complete->optional, TRUE, data_set);
add_hash_param(notify->meta, "notify_type", "post");
add_hash_param(notify->meta, "notify_operation", action->task);
if(clone_data->notify_confirm) {
add_hash_param(notify->meta, "notify_confirm", "yes");
} else {
add_hash_param(notify->meta, "notify_confirm", "no");
}
/* action_complete before post_notify */
custom_action_order(
rsc, NULL, action_complete,
rsc, NULL, notify,
pe_order_implies_right, data_set);
/* create post_notify_complete */
notify_key = generate_notify_key(
rsc->id, "confirmed-post", action->task);
notify_complete = custom_action(rsc, notify_key,
CRMD_ACTION_NOTIFIED, NULL,
action->optional, TRUE, data_set);
add_hash_param(notify_complete->meta, "notify_type", "pre");
add_hash_param(notify_complete->meta, "notify_operation", action->task);
if(clone_data->notify_confirm) {
add_hash_param(notify->meta, "notify_confirm", "yes");
} else {
add_hash_param(notify->meta, "notify_confirm", "no");
}
notify->pseudo = TRUE;
notify->runnable = TRUE;
notify->priority = INFINITY;
notify->runnable = action_complete->runnable;
notify_complete->pseudo = TRUE;
notify_complete->runnable = TRUE;
notify_complete->priority = INFINITY;
notify_complete->runnable = action_complete->runnable;
/* post_notify before post_notify_complete */
custom_action_order(
rsc, NULL, notify,
rsc, NULL, notify_complete,
pe_order_implies_left, data_set);
action->post_notify = notify;
action->post_notified = notify_complete;
if(safe_str_eq(action->task, CRMD_ACTION_STOP)) {
/* post_notify_complete before start */
custom_action_order(
rsc, NULL, notify_complete,
rsc, start_key(rsc), NULL,
pe_order_optional, data_set);
} else if(safe_str_eq(action->task, CRMD_ACTION_START)) {
/* post_notify_complete before promote */
custom_action_order(
rsc, NULL, notify_complete,
rsc, promote_key(rsc), NULL,
pe_order_optional, data_set);
} else if(safe_str_eq(action->task, CRMD_ACTION_DEMOTE)) {
/* post_notify_complete before promote */
custom_action_order(
rsc, NULL, notify_complete,
rsc, stop_key(rsc), NULL,
pe_order_optional, data_set);
}
}
void
child_starting_constraints(
clone_variant_data_t *clone_data,
resource_t *rsc, resource_t *child, resource_t *last,
pe_working_set_t *data_set)
{
if(child == NULL && last == NULL) {
crm_debug("%s has no active children", rsc->id);
return;
}
if(child != NULL) {
order_start_start(rsc, child, pe_order_runnable_left);
custom_action_order(
child, start_key(child), NULL,
rsc, started_key(rsc), NULL,
pe_order_optional, data_set);
}
if(clone_data->ordered) {
if(child == NULL) {
/* last child start before global started */
custom_action_order(
last, start_key(last), NULL,
rsc, started_key(rsc), NULL,
pe_order_runnable_left, data_set);
} else if(last == NULL) {
/* global start before first child start */
order_start_start(
rsc, child, pe_order_implies_left);
} else {
/* child/child relative start */
order_start_start(last, child, pe_order_implies_left);
}
}
}
void
child_stopping_constraints(
clone_variant_data_t *clone_data,
resource_t *rsc, resource_t *child, resource_t *last,
pe_working_set_t *data_set)
{
if(child == NULL && last == NULL) {
crm_debug("%s has no active children", rsc->id);
return;
}
if(child != NULL) {
order_stop_stop(rsc, child, pe_order_shutdown);
custom_action_order(
child, stop_key(child), NULL,
rsc, stopped_key(rsc), NULL,
pe_order_optional, data_set);
}
if(clone_data->ordered) {
if(last == NULL) {
/* first child stop before global stopped */
custom_action_order(
child, stop_key(child), NULL,
rsc, stopped_key(rsc), NULL,
pe_order_runnable_left, data_set);
} else if(child == NULL) {
/* global stop before last child stop */
order_stop_stop(
rsc, last, pe_order_implies_left);
} else {
/* child/child relative stop */
order_stop_stop(child, last, pe_order_implies_left);
}
}
}
void
clone_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
{
resource_t *last_rsc = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
native_internal_constraints(rsc, data_set);
/* global stop before stopped */
custom_action_order(
rsc, stop_key(rsc), NULL,
rsc, stopped_key(rsc), NULL,
pe_order_runnable_left, data_set);
/* global start before started */
custom_action_order(
rsc, start_key(rsc), NULL,
rsc, started_key(rsc), NULL,
pe_order_runnable_left, data_set);
/* global stopped before start */
custom_action_order(
rsc, stopped_key(rsc), NULL,
rsc, start_key(rsc), NULL,
pe_order_optional, data_set);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->cmds->internal_constraints(child_rsc, data_set);
child_starting_constraints(
clone_data, rsc, child_rsc, last_rsc, data_set);
child_stopping_constraints(
clone_data, rsc, child_rsc, last_rsc, data_set);
last_rsc = child_rsc;
);
}
-static resource_t*
-find_compatible_child(resource_t *local_child, resource_t *rsc)
+resource_t*
+find_compatible_child(
+ resource_t *local_child, resource_t *rsc, enum rsc_role_e filter)
{
node_t *local_node = NULL;
node_t *node = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
local_node = local_child->allocated_to;
if(local_node == NULL) {
crm_debug("Can't colocate unrunnable child %s with %s",
local_child->id, rsc->id);
return NULL;
}
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
node = child_rsc->allocated_to;
+
+ if(filter != RSC_ROLE_UNKNOWN
+ && child_rsc->next_role != filter) {
+ crm_debug_2("Filtered %s", child_rsc->id);
+ continue;
+ }
+
if(node->details == local_node->details) {
crm_info("Colocating %s with %s on %s",
local_child->id, child_rsc->id, node->details->uname);
return child_rsc;
}
);
crm_debug("Can't colocate child %s with %s",
local_child->id, rsc->id);
return NULL;
}
void clone_rsc_colocation_lh(
resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
{
gboolean do_interleave = FALSE;
resource_t *rsc = constraint->rsc_lh;
clone_variant_data_t *clone_data = NULL;
clone_variant_data_t *clone_data_rh = NULL;
if(rsc == NULL) {
pe_err("rsc_lh was NULL for %s", constraint->id);
return;
} else if(constraint->rsc_rh == NULL) {
pe_err("rsc_rh was NULL for %s", constraint->id);
return;
} else {
crm_debug_4("Processing constraints from %s", rsc->id);
}
get_clone_variant_data(clone_data, rsc);
- if(constraint->rsc_rh->variant == pe_clone) {
+ if(constraint->rsc_rh->variant == pe_clone
+ || constraint->rsc_rh->variant == pe_master) {
get_clone_variant_data(
clone_data_rh, constraint->rsc_rh);
if(clone_data->clone_node_max
!= clone_data_rh->clone_node_max) {
crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION
" %s and %s because"
" they do not support the same number of"
" resources per node",
constraint->rsc_lh->id, constraint->rsc_rh->id);
/* only the LHS side needs to be labeled as interleave */
} else if(clone_data->interleave) {
do_interleave = TRUE;
} else if(constraint->score >= INFINITY) {
GListPtr lhs = NULL, rhs = NULL;
lhs = rsc_lh->allowed_nodes;
slist_iter(
child_rsc, resource_t, rsc_rh->children, lpc,
if(child_rsc->allocated_to != NULL) {
rhs = g_list_append(rhs, child_rsc->allocated_to);
}
);
rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);
pe_free_shallow_adv(rhs, FALSE);
pe_free_shallow(lhs);
return;
}
} else if(constraint->score >= INFINITY) {
crm_config_err("Manditory co-location of clones (%s) with other"
" non-clone (%s) resources is not supported",
rsc_lh->id, rsc_rh->id);
return;
}
if(do_interleave) {
resource_t *rh_child = NULL;
slist_iter(lh_child, resource_t, rsc->children, lpc,
CRM_ASSERT(lh_child != NULL);
- rh_child = find_compatible_child(lh_child, rsc_rh);
+ rh_child = find_compatible_child(
+ lh_child, rsc_rh, RSC_ROLE_UNKNOWN);
if(rh_child == NULL) {
continue;
}
lh_child->cmds->rsc_colocation_lh(
lh_child, rh_child, constraint);
);
return;
}
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->cmds->rsc_colocation_lh(child_rsc, constraint->rsc_rh, constraint);
);
}
void clone_rsc_colocation_rh(
resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
{
clone_variant_data_t *clone_data = NULL;
CRM_CHECK(rsc_lh != NULL, return);
CRM_CHECK(rsc_lh->variant == pe_native, return);
get_clone_variant_data(clone_data, rsc_rh);
crm_debug_3("Processing constraint %s: %d", constraint->id, constraint->score);
if(rsc_rh == NULL) {
pe_err("rsc_rh was NULL for %s", constraint->id);
return;
} else if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
crm_debug_3("%s is still provisional", rsc_rh->id);
return;
} else if(constraint->score >= INFINITY) {
GListPtr lhs = NULL, rhs = NULL;
lhs = rsc_lh->allowed_nodes;
slist_iter(
child_rsc, resource_t, rsc_rh->children, lpc,
if(child_rsc->allocated_to != NULL) {
rhs = g_list_append(rhs, child_rsc->allocated_to);
}
);
rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);
pe_free_shallow_adv(rhs, FALSE);
pe_free_shallow(lhs);
return;
}
slist_iter(
child_rsc, resource_t, rsc_rh->children, lpc,
child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
);
}
void clone_rsc_order_lh(resource_t *rsc, order_constraint_t *order, pe_working_set_t *data_set)
{
resource_t *r1 = NULL;
resource_t *r2 = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_debug_2("%s->%s", order->lh_action_task, order->rh_action_task);
r1 = uber_parent(rsc);
r2 = uber_parent(order->rh_rsc);
if(r1 == r2) {
native_rsc_order_lh(rsc, order, data_set);
return;
}
#if 0
if(order->type != pe_order_optional) {
crm_debug("Upgraded ordering constraint %d - 0x%.6x", order->id, order->type);
native_rsc_order_lh(rsc, order, data_set);
}
#endif
if(order->type & pe_order_implies_left) {
if(rsc->variant == order->rh_rsc->variant) {
crm_debug_2("Clone-to-clone ordering: %s -> %s 0x%.6x",
order->lh_action_task, order->rh_action_task, order->type);
/* stop instances on the same nodes as stopping RHS instances */
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
native_rsc_order_lh(child_rsc, order, data_set);
);
} else {
/* stop everything */
crm_debug_2("Clone-to-* ordering: %s -> %s 0x%.6x",
order->lh_action_task, order->rh_action_task, order->type);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
native_rsc_order_lh(child_rsc, order, data_set);
);
}
}
convert_non_atomic_task(rsc, order);
native_rsc_order_lh(rsc, order, data_set);
}
void clone_rsc_order_rh(
action_t *lh_action, resource_t *rsc, order_constraint_t *order)
{
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_debug_2("%s->%s", lh_action->uuid, order->rh_action_task);
native_rsc_order_rh(lh_action, rsc, order);
}
void clone_rsc_location(resource_t *rsc, rsc_to_node_t *constraint)
{
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_debug_3("Processing location constraint %s for %s",
constraint->id, rsc->id);
native_rsc_location(rsc, constraint);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->cmds->rsc_location(child_rsc, constraint);
);
}
static gint
sort_notify_entries(gconstpointer a, gconstpointer b)
{
int tmp;
const notify_entry_t *entry_a = a;
const notify_entry_t *entry_b = b;
if(entry_a == NULL && entry_b == NULL) { return 0; }
if(entry_a == NULL) { return 1; }
if(entry_b == NULL) { return -1; }
if(entry_a->rsc == NULL && entry_b->rsc == NULL) { return 0; }
if(entry_a->rsc == NULL) { return 1; }
if(entry_b->rsc == NULL) { return -1; }
tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
if(tmp != 0) {
return tmp;
}
if(entry_a->node == NULL && entry_b->node == NULL) { return 0; }
if(entry_a->node == NULL) { return 1; }
if(entry_b->node == NULL) { return -1; }
return strcmp(entry_a->node->details->id, entry_b->node->details->id);
}
static void
expand_list(GListPtr list, int clones,
char **rsc_list, char **node_list, char **uuid_list)
{
const char *uname = NULL;
const char *rsc_id = NULL;
const char *last_rsc_id = NULL;
CRM_CHECK(list != NULL, return);
if(rsc_list) {
CRM_CHECK(*rsc_list == NULL, *rsc_list = NULL);
}
if(node_list) {
CRM_CHECK(*node_list == NULL, *node_list = NULL);
}
slist_iter(entry, notify_entry_t, list, lpc,
CRM_CHECK(entry != NULL, continue);
CRM_CHECK(entry->rsc != NULL, continue);
CRM_CHECK(entry->node != NULL, continue);
rsc_id = entry->rsc->id;
uname = entry->node->details->uname;
CRM_ASSERT(uname != NULL);
CRM_ASSERT(rsc_id != NULL);
/* filter dups */
if(safe_str_eq(rsc_id, last_rsc_id)) {
continue;
}
last_rsc_id = rsc_id;
if(rsc_list != NULL) {
int existing_len = 0;
int len = 2 + strlen(rsc_id); /* +1 space, +1 EOS */
if(rsc_list && *rsc_list) {
existing_len = strlen(*rsc_list);
}
crm_debug_5("Adding %s (%dc) at offset %d",
rsc_id, len-2, existing_len);
crm_realloc(*rsc_list, len + existing_len);
sprintf(*rsc_list + existing_len, "%s ", rsc_id);
}
if(node_list != NULL) {
int existing_len = 0;
int len = 2 + strlen(uname);
if(node_list && *node_list) {
existing_len = strlen(*node_list);
}
crm_debug_5("Adding %s (%dc) at offset %d",
uname, len-2, existing_len);
crm_realloc(*node_list, len + existing_len);
sprintf(*node_list + existing_len, "%s ", uname);
}
);
}
void clone_expand(resource_t *rsc, pe_working_set_t *data_set)
{
char *rsc_list = NULL;
char *node_list = NULL;
char *uuid_list = NULL;
notify_data_t *n_data = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_malloc0(n_data, sizeof(notify_data_t));
n_data->keys = g_hash_table_new_full(
g_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
crm_debug_2("Processing actions from %s", rsc->id);
if(is_set(rsc->flags, pe_rsc_notify)) {
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
slist_iter(
op, action_t, rsc->actions, lpc2,
child_rsc->cmds->create_notify_element(
child_rsc, op, n_data, data_set);
);
);
}
/* expand the notify data */
if(is_set(rsc->flags, pe_rsc_notify) && n_data->stop) {
n_data->stop = g_list_sort(
n_data->stop, sort_notify_entries);
rsc_list = NULL; node_list = NULL;
expand_list(n_data->stop, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_stop_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_stop_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->start) {
n_data->start = g_list_sort(
n_data->start, sort_notify_entries);
rsc_list = NULL; node_list = NULL;
expand_list(n_data->start, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_start_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_start_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->demote) {
n_data->demote = g_list_sort(
n_data->demote, sort_notify_entries);
rsc_list = NULL; node_list = NULL;
expand_list(n_data->demote, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_demote_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_demote_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->promote) {
n_data->promote = g_list_sort(
n_data->promote, sort_notify_entries);
rsc_list = NULL; node_list = NULL; uuid_list = NULL;
expand_list(n_data->promote, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_promote_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_promote_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->active) {
n_data->active = g_list_sort(
n_data->active, sort_notify_entries);
rsc_list = NULL; node_list = NULL; uuid_list = NULL;
expand_list(n_data->active, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_active_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_active_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->slave) {
n_data->slave = g_list_sort(
n_data->slave, sort_notify_entries);
rsc_list = NULL; node_list = NULL; uuid_list = NULL;
expand_list(n_data->slave, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_slave_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_slave_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->master) {
n_data->master = g_list_sort(
n_data->master, sort_notify_entries);
rsc_list = NULL; node_list = NULL; uuid_list = NULL;
expand_list(n_data->master, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_master_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_master_uname"), node_list);
}
if(is_set(rsc->flags, pe_rsc_notify) && n_data->inactive) {
n_data->inactive = g_list_sort(
n_data->inactive, sort_notify_entries);
rsc_list = NULL; node_list = NULL; uuid_list = NULL;
expand_list(n_data->inactive, clone_data->clone_max,
&rsc_list, &node_list, &uuid_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_inactive_resource"), rsc_list);
g_hash_table_insert(
n_data->keys,
crm_strdup("notify_inactive_uname"), node_list);
}
/* yes, we DO need this second loop */
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->cmds->expand(child_rsc, data_set);
);
/* slist_iter( */
/* action, action_t, rsc->actions, lpc2, */
/* if(safe_str_eq(action->task, CRMD_ACTION_NOTIFY)) { */
/* action->meta_xml = notify_xml; */
/* } */
/* ); */
native_expand(rsc, data_set);
/* destroy the notify_data */
pe_free_shallow(n_data->stop);
pe_free_shallow(n_data->start);
pe_free_shallow(n_data->demote);
pe_free_shallow(n_data->promote);
pe_free_shallow(n_data->master);
pe_free_shallow(n_data->slave);
pe_free_shallow(n_data->active);
pe_free_shallow(n_data->inactive);
g_hash_table_destroy(n_data->keys);
crm_free(n_data);
}
static gint sort_rsc_id(gconstpointer a, gconstpointer b)
{
const resource_t *resource1 = (const resource_t*)a;
const resource_t *resource2 = (const resource_t*)b;
CRM_ASSERT(resource1 != NULL);
CRM_ASSERT(resource2 != NULL);
return strcmp(resource1->id, resource2->id);
}
gboolean
clone_create_probe(resource_t *rsc, node_t *node, action_t *complete,
gboolean force, pe_working_set_t *data_set)
{
gboolean any_created = FALSE;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
rsc->children = g_list_sort(rsc->children, sort_rsc_id);
if(is_not_set(rsc->flags, pe_rsc_unique)
&& clone_data->clone_node_max == 1) {
/* only look for one copy */
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
if(pe_find_node_id(child_rsc->running_on, node->details->id)) {
return child_rsc->cmds->create_probe(
child_rsc, node, complete, force, data_set);
}
);
}
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
if(child_rsc->cmds->create_probe(
child_rsc, node, complete, force, data_set)) {
any_created = TRUE;
}
if(any_created
&& is_not_set(rsc->flags, pe_rsc_unique)
&& clone_data->clone_node_max == 1) {
/* only look for one copy (clone :0) */
break;
}
);
return any_created;
}
diff --git a/crm/pengine/master.c b/crm/pengine/master.c
index 474e9830cf..4e78cc79d3 100644
--- a/crm/pengine/master.c
+++ b/crm/pengine/master.c
@@ -1,737 +1,762 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <lha_internal.h>
#include <crm/msg_xml.h>
#include <allocate.h>
#include <lib/crm/pengine/utils.h>
#include <utils.h>
#define VARIANT_CLONE 1
#include <lib/crm/pengine/variant.h>
extern gint sort_clone_instance(gconstpointer a, gconstpointer b);
extern void clone_create_notifications(
resource_t *rsc, action_t *action, action_t *action_complete,
pe_working_set_t *data_set);
extern int master_score(resource_t *rsc, node_t *node, int not_set_value);
static void
child_promoting_constraints(
clone_variant_data_t *clone_data, enum pe_ordering type,
resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set)
{
/* if(clone_data->ordered */
/* || clone_data->self->restart_type == pe_restart_restart) { */
/* type = pe_order_implies_left; */
/* } */
if(child == NULL) {
if(clone_data->ordered && last != NULL) {
crm_debug_4("Ordered version (last node)");
/* last child promote before promoted started */
custom_action_order(
last, promote_key(last), NULL,
rsc, promoted_key(rsc), NULL,
type, data_set);
}
} else if(clone_data->ordered) {
crm_debug_4("Ordered version");
if(last == NULL) {
/* global promote before first child promote */
last = rsc;
} /* else: child/child relative promote */
order_start_start(last, child, type);
custom_action_order(
last, promote_key(last), NULL,
child, promote_key(child), NULL,
type, data_set);
} else {
crm_debug_4("Un-ordered version");
/* child promote before global promoted */
custom_action_order(
child, promote_key(child), NULL,
rsc, promoted_key(rsc), NULL,
type, data_set);
/* global promote before child promote */
custom_action_order(
rsc, promote_key(rsc), NULL,
child, promote_key(child), NULL,
type, data_set);
}
}
static void
child_demoting_constraints(
clone_variant_data_t *clone_data, enum pe_ordering type,
resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set)
{
/* if(clone_data->ordered */
/* || clone_data->self->restart_type == pe_restart_restart) { */
/* type = pe_order_implies_left; */
/* } */
if(child == NULL) {
if(clone_data->ordered && last != NULL) {
crm_debug_4("Ordered version (last node)");
/* global demote before first child demote */
custom_action_order(
rsc, demote_key(rsc), NULL,
last, demote_key(last), NULL,
pe_order_implies_left, data_set);
}
} else if(clone_data->ordered && last != NULL) {
crm_debug_4("Ordered version");
/* child/child relative demote */
custom_action_order(child, demote_key(child), NULL,
last, demote_key(last), NULL,
type, data_set);
} else if(clone_data->ordered) {
crm_debug_4("Ordered version (1st node)");
/* first child stop before global stopped */
custom_action_order(
child, demote_key(child), NULL,
rsc, demoted_key(rsc), NULL,
type, data_set);
} else {
crm_debug_4("Un-ordered version");
/* child demote before global demoted */
custom_action_order(
child, demote_key(child), NULL,
rsc, demoted_key(rsc), NULL,
type, data_set);
/* global demote before child demote */
custom_action_order(
rsc, demote_key(rsc), NULL,
child, demote_key(child), NULL,
type, data_set);
}
}
static void
master_update_pseudo_status(
resource_t *child, gboolean *demoting, gboolean *promoting)
{
CRM_ASSERT(demoting != NULL);
CRM_ASSERT(promoting != NULL);
slist_iter(
action, action_t, child->actions, lpc,
if(*promoting && *demoting) {
return;
} else if(action->optional) {
continue;
} else if(safe_str_eq(CRMD_ACTION_DEMOTE, action->task)) {
*demoting = TRUE;
} else if(safe_str_eq(CRMD_ACTION_PROMOTE, action->task)) {
*promoting = TRUE;
}
);
}
#define apply_master_location(list) \
slist_iter( \
cons, rsc_to_node_t, list, lpc2, \
cons_node = NULL; \
if(cons->role_filter == RSC_ROLE_MASTER) { \
crm_debug("Applying %s to %s", \
cons->id, child_rsc->id); \
cons_node = pe_find_node_id( \
cons->node_list_rh, chosen->details->id); \
} \
if(cons_node != NULL) { \
int new_priority = merge_weights( \
child_rsc->priority, cons_node->weight); \
crm_debug("\t%s: %d->%d", child_rsc->id, \
child_rsc->priority, new_priority); \
child_rsc->priority = new_priority; \
} \
);
-#define apply_master_colocation(list) \
- slist_iter( \
- cons, rsc_colocation_t, list, lpc2, \
- cons_node = cons->rsc_lh->allocated_to; \
- if(cons->role_lh == RSC_ROLE_MASTER \
- && cons_node != NULL \
- && chosen->details == cons_node->details) { \
- int new_priority = merge_weights( \
- child_rsc->priority, cons->score); \
- crm_debug("Applying %s to %s", \
- cons->id, child_rsc->id); \
- crm_debug("\t%s: %d->%d", child_rsc->id, \
- child_rsc->priority, new_priority); \
- child_rsc->priority = new_priority; \
- } \
- );
-
static node_t *
can_be_master(resource_t *rsc)
{
node_t *node = NULL;
node_t *local_node = NULL;
clone_variant_data_t *clone_data = NULL;
int level = LOG_DEBUG_2;
node = rsc->allocated_to;
if(rsc->priority < 0) {
do_crm_log(level, "%s cannot be master: preference: %d",
rsc->id, rsc->priority);
return NULL;
} else if(node == NULL) {
do_crm_log(level, "%s cannot be master: not allocated",
rsc->id);
return NULL;
} else if(can_run_resources(node) == FALSE) {
do_crm_log(level, "Node cant run any resources: %s",
node->details->uname);
return NULL;
}
get_clone_variant_data(clone_data, rsc->parent);
local_node = pe_find_node_id(
rsc->parent->allowed_nodes, node->details->id);
if(local_node == NULL) {
crm_err("%s cannot run on %s: node not allowed",
rsc->id, node->details->uname);
return NULL;
} else if(local_node->count < clone_data->master_node_max) {
return local_node;
} else {
do_crm_log(level, "%s cannot be master on %s: node full",
rsc->id, node->details->uname);
}
return NULL;
}
static gint sort_master_instance(gconstpointer a, gconstpointer b)
{
int rc;
const resource_t *resource1 = (const resource_t*)a;
const resource_t *resource2 = (const resource_t*)b;
CRM_ASSERT(resource1 != NULL);
CRM_ASSERT(resource2 != NULL);
rc = sort_rsc_index(a, b);
if( rc != 0 ) {
return rc;
}
if(resource1->role > resource2->role) {
return -1;
} else if(resource1->role < resource2->role) {
return 1;
}
return sort_clone_instance(a, b);
}
static void master_promotion_order(resource_t *rsc)
{
node_t *node = NULL;
node_t *chosen = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if(clone_data->merged_master_weights) {
return;
}
clone_data->merged_master_weights = TRUE;
crm_info("Merging weights for %s", rsc->id);
slist_iter(
child, resource_t, rsc->children, lpc,
crm_debug_2("%s: %d", child->id, child->sort_index);
);
dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes);
#if 1
slist_iter(
child, resource_t, rsc->children, lpc,
chosen = child->allocated_to;
if(chosen == NULL || child->sort_index < 0) {
crm_debug_3("Skipping %s", child->id);
continue;
}
node = (node_t*)pe_find_node_id(
rsc->allowed_nodes, chosen->details->id);
CRM_ASSERT(node != NULL);
node->weight = merge_weights(child->sort_index, node->weight);
);
dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes);
#endif
slist_iter(
constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
if(constraint->role_rh == RSC_ROLE_MASTER) {
rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
constraint->score/INFINITY, TRUE);
}
);
dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes);
/* write them back and sort */
slist_iter(
child, resource_t, rsc->children, lpc,
chosen = child->allocated_to;
if(chosen == NULL || child->sort_index < 0) {
crm_debug_2("%s: %d", child->id, child->sort_index);
continue;
}
node = (node_t*)pe_find_node_id(
rsc->allowed_nodes, chosen->details->id);
CRM_ASSERT(node != NULL);
child->sort_index = node->weight;
crm_debug_2("%s: %d", child->id, child->sort_index);
);
rsc->children = g_list_sort(rsc->children, sort_master_instance);
}
int
master_score(resource_t *rsc, node_t *node, int not_set_value)
{
char *attr_name;
const char *attr_value;
int score = not_set_value, len = 0;
len = 8 + strlen(rsc->id);
crm_malloc0(attr_name, len);
sprintf(attr_name, "master-%s", rsc->id);
crm_debug_3("looking for %s on %s", attr_name,
node->details->uname);
attr_value = g_hash_table_lookup(
node->details->attrs, attr_name);
if(attr_value == NULL) {
crm_free(attr_name);
len = 8 + strlen(rsc->long_name);
crm_malloc0(attr_name, len);
sprintf(attr_name, "master-%s", rsc->long_name);
crm_debug_3("looking for %s on %s", attr_name,
node->details->uname);
attr_value = g_hash_table_lookup(
node->details->attrs, attr_name);
}
if(attr_value != NULL) {
crm_debug_2("%s[%s] = %s", attr_name,
node->details->uname, crm_str(attr_value));
score = char2score(attr_value);
}
crm_free(attr_name);
return score;
}
#define max(a, b) a<b?b:a
static void
apply_master_prefs(resource_t *rsc)
{
int score, new_score;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if(clone_data->applied_master_prefs) {
/* Make sure we only do this once */
return;
}
clone_data->applied_master_prefs = TRUE;
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
slist_iter(
node, node_t, child_rsc->allowed_nodes, lpc,
if(can_run_resources(node) == FALSE) {
/* This node will never be promoted to master,
* so don't apply the master score as that may
* lead to clone shuffling
*/
continue;
}
score = master_score(child_rsc, node, 0);
new_score = merge_weights(node->weight, score);
if(new_score != node->weight) {
crm_debug("\t%s: Updating preference for %s (%d->%d)",
child_rsc->id, node->details->uname, node->weight, new_score);
node->weight = new_score;
}
new_score = max(child_rsc->priority, score);
if(new_score != child_rsc->priority) {
crm_debug("\t%s: Updating priority (%d->%d)",
child_rsc->id, child_rsc->priority, new_score);
child_rsc->priority = new_score;
}
);
);
}
node_t *
master_color(resource_t *rsc, pe_working_set_t *data_set)
{
int promoted = 0;
node_t *chosen = NULL;
node_t *cons_node = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
apply_master_prefs(rsc);
clone_color(rsc, data_set);
/* count now tracks the number of masters allocated */
slist_iter(node, node_t, rsc->allowed_nodes, lpc,
node->count = 0;
);
/*
* assign priority
*/
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
crm_debug_2("Assigning priority for %s", child_rsc->id);
if(child_rsc->role == RSC_ROLE_STARTED) {
child_rsc->role = RSC_ROLE_SLAVE;
}
chosen = child_rsc->allocated_to;
if(chosen == NULL) {
continue;
}
switch(child_rsc->next_role) {
case RSC_ROLE_STARTED:
CRM_CHECK(chosen != NULL, break);
/*
* Default to -1 if no value is set
*
* This allows master locations to be specified
* based solely on rsc_location constraints,
* but prevents anyone from being promoted if
* neither a constraint nor a master-score is present
*/
child_rsc->priority = master_score(child_rsc, chosen, -1);
break;
case RSC_ROLE_SLAVE:
case RSC_ROLE_STOPPED:
child_rsc->priority = -INFINITY;
break;
case RSC_ROLE_MASTER:
/* the only reason we should be here is if
* we're re-creating actions after a stonith
*/
promoted++;
break;
default:
CRM_CHECK(FALSE/* unhandled */,
crm_err("Unknown resource role: %d for %s",
child_rsc->next_role, child_rsc->id));
}
apply_master_location(child_rsc->rsc_location);
apply_master_location(rsc->rsc_location);
- apply_master_colocation(rsc->rsc_cons);
- apply_master_colocation(child_rsc->rsc_cons);
+ slist_iter(
+ cons, rsc_colocation_t, child_rsc->rsc_cons, lpc2,
+ child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons);
+ );
+
child_rsc->sort_index = child_rsc->priority;
crm_debug_2("Assigning priority for %s: %d", child_rsc->id, child_rsc->priority);
if(child_rsc->next_role == RSC_ROLE_MASTER) {
child_rsc->sort_index = INFINITY;
}
);
master_promotion_order(rsc);
/* mark the first N as masters */
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
chosen = NULL;
crm_debug_2("Processing %s", child_rsc->id);
if(promoted < clone_data->master_max) {
chosen = can_be_master(child_rsc);
}
if(chosen == NULL) {
if(child_rsc->next_role == RSC_ROLE_STARTED) {
child_rsc->next_role = RSC_ROLE_SLAVE;
}
continue;
}
chosen->count++;
crm_info("Promoting %s", child_rsc->id);
child_rsc->next_role = RSC_ROLE_MASTER;
clone_data->masters_allocated++;
promoted++;
add_hash_param(child_rsc->parameters, crm_meta_name("role"),
role2text(child_rsc->next_role));
);
crm_info("%s: Promoted %d instances of a possible %d to master",
rsc->id, promoted, clone_data->master_max);
return NULL;
}
void master_create_actions(resource_t *rsc, pe_working_set_t *data_set)
{
action_t *action = NULL;
action_t *action_complete = NULL;
gboolean any_promoting = FALSE;
gboolean any_demoting = FALSE;
resource_t *last_promote_rsc = NULL;
resource_t *last_demote_rsc = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
crm_debug("Creating actions for %s", rsc->id);
/* create actions as normal */
clone_create_actions(rsc, data_set);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
gboolean child_promoting = FALSE;
gboolean child_demoting = FALSE;
crm_debug_2("Creating actions for %s", child_rsc->id);
child_rsc->cmds->create_actions(child_rsc, data_set);
master_update_pseudo_status(
child_rsc, &child_demoting, &child_promoting);
any_demoting = any_demoting || child_demoting;
any_promoting = any_promoting || child_promoting;
);
/* promote */
action = promote_action(rsc, NULL, !any_promoting);
action_complete = custom_action(
rsc, promoted_key(rsc),
CRMD_ACTION_PROMOTED, NULL, !any_promoting, TRUE, data_set);
action->pseudo = TRUE;
action->runnable = FALSE;
action_complete->pseudo = TRUE;
action_complete->runnable = FALSE;
action_complete->priority = INFINITY;
if(clone_data->masters_allocated > 0) {
action->runnable = TRUE;
action_complete->runnable = TRUE;
}
child_promoting_constraints(clone_data, pe_order_optional,
rsc, NULL, last_promote_rsc, data_set);
clone_create_notifications(rsc, action, action_complete, data_set);
/* demote */
action = demote_action(rsc, NULL, !any_demoting);
action_complete = custom_action(
rsc, demoted_key(rsc),
CRMD_ACTION_DEMOTED, NULL, !any_demoting, TRUE, data_set);
action_complete->priority = INFINITY;
action->pseudo = TRUE;
action->runnable = TRUE;
action_complete->pseudo = TRUE;
action_complete->runnable = TRUE;
child_demoting_constraints(clone_data, pe_order_optional,
rsc, NULL, last_demote_rsc, data_set);
clone_create_notifications(rsc, action, action_complete, data_set);
/* restore the correct priority */
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
child_rsc->priority = rsc->priority;
);
}
void
master_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
{
resource_t *last_rsc = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
clone_internal_constraints(rsc, data_set);
/* global stopped before start */
custom_action_order(
rsc, stopped_key(rsc), NULL,
rsc, start_key(rsc), NULL,
pe_order_optional, data_set);
/* global stopped before promote */
custom_action_order(
rsc, stopped_key(rsc), NULL,
rsc, promote_key(rsc), NULL,
pe_order_optional, data_set);
/* global demoted before start */
custom_action_order(
rsc, demoted_key(rsc), NULL,
rsc, start_key(rsc), NULL,
pe_order_optional, data_set);
/* global started before promote */
custom_action_order(
rsc, started_key(rsc), NULL,
rsc, promote_key(rsc), NULL,
pe_order_optional, data_set);
/* global demoted before stop */
custom_action_order(
rsc, demoted_key(rsc), NULL,
rsc, stop_key(rsc), NULL,
pe_order_optional, data_set);
/* global demote before demoted */
custom_action_order(
rsc, demote_key(rsc), NULL,
rsc, demoted_key(rsc), NULL,
pe_order_optional, data_set);
/* global demoted before promote */
custom_action_order(
rsc, demoted_key(rsc), NULL,
rsc, promote_key(rsc), NULL,
pe_order_optional, data_set);
slist_iter(
child_rsc, resource_t, rsc->children, lpc,
/* child demote before promote */
custom_action_order(
child_rsc, demote_key(child_rsc), NULL,
child_rsc, promote_key(child_rsc), NULL,
pe_order_optional, data_set);
child_promoting_constraints(clone_data, pe_order_optional,
rsc, child_rsc, last_rsc, data_set);
child_demoting_constraints(clone_data, pe_order_optional,
rsc, child_rsc, last_rsc, data_set);
last_rsc = child_rsc;
);
}
+extern resource_t *find_compatible_child(resource_t *local_child, resource_t *rsc, enum rsc_role_e filter);
+static void node_list_update_one(GListPtr list1, node_t *other, int score)
+{
+ node_t *node = NULL;
+
+ if(other == NULL) {
+ return;
+ }
+
+ node = (node_t*)pe_find_node_id(list1, other->details->id);
+
+ if(node != NULL) {
+ crm_debug_2("%s: %d + %d",
+ node->details->uname, node->weight, other->weight);
+ node->weight = merge_weights(node->weight, score);
+ }
+}
+
void master_rsc_colocation_rh(
- resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
+ resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
{
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc_rh);
CRM_CHECK(rsc_rh != NULL, return);
if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
return;
} else if(constraint->role_rh == RSC_ROLE_UNKNOWN) {
crm_debug_3("Handling %s as a clone colocation", constraint->id);
clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
return;
}
CRM_CHECK(rsc_lh != NULL, return);
CRM_CHECK(rsc_lh->variant == pe_native, return);
crm_debug_2("Processing constraint %s: %d", constraint->id, constraint->score);
if(constraint->score < INFINITY) {
slist_iter(
child_rsc, resource_t, rsc_rh->children, lpc,
child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
);
- } else {
+ } else if(is_set(rsc_lh->flags, pe_rsc_provisional)) {
GListPtr lhs = NULL, rhs = NULL;
lhs = rsc_lh->allowed_nodes;
-
+
slist_iter(
child_rsc, resource_t, rsc_rh->children, lpc,
crm_debug_3("Processing: %s", child_rsc->id);
if(child_rsc->allocated_to != NULL
&& child_rsc->next_role == constraint->role_rh) {
- crm_debug_3("Applying: %s %s", child_rsc->id, role2text(child_rsc->next_role));
- rhs = g_list_append(rhs, child_rsc->allocated_to);
+ crm_debug_3("Applying: %s %s", child_rsc->id,
+ role2text(child_rsc->next_role));
+ node_list_update_one(rsc_lh->allowed_nodes, child_rsc->allocated_to, constraint->score);
+ rhs = g_list_append(rhs, child_rsc->allocated_to);
}
);
-
- rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);
+
+ /* Only do this if its not a master-master colocation
+ * Doing this unconditionally would prevent the slaves from being started
+ */
+ if(constraint->role_lh != RSC_ROLE_MASTER
+ || constraint->role_rh != RSC_ROLE_MASTER) {
+ rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);
+ pe_free_shallow(lhs);
+ }
pe_free_shallow_adv(rhs, FALSE);
- pe_free_shallow(lhs);
+
+ } else if(constraint->role_lh == RSC_ROLE_MASTER) {
+ resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh);
+ if(rh_child == NULL && constraint->score >= INFINITY) {
+ crm_debug_2("%s can't be promoted %s", rsc_lh->id, constraint->id);
+ rsc_lh->priority = -INFINITY;
+
+ } else if(rh_child != NULL) {
+ int new_priority = merge_weights(rsc_lh->priority, constraint->score);
+ crm_debug("Applying %s to %s", constraint->id, rsc_lh->id);
+ crm_debug("\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority);
+ rsc_lh->priority = new_priority;
+ }
}
return;
}
diff --git a/crm/pengine/regression.sh b/crm/pengine/regression.sh
index 704331a0e3..0cc1565dd2 100755
--- a/crm/pengine/regression.sh
+++ b/crm/pengine/regression.sh
@@ -1,269 +1,270 @@
#!/bin/bash
# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
if [ -x /usr/bin/valgrind ]; then
export G_SLICE=always-malloc
VALGRIND_CMD="valgrind -q --show-reachable=yes --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=./ptest.supp"
fi
. regression.core.sh
create_mode="true"
echo Generating test outputs for these tests...
# do_test
echo Done.
echo ""
echo Performing the following tests...
create_mode="false"
echo ""
do_test simple1 "Offline "
do_test simple2 "Start "
do_test simple3 "Start 2 "
do_test simple4 "Start Failed"
do_test simple6 "Stop Start "
do_test simple7 "Shutdown "
#do_test simple8 "Stonith "
#do_test simple9 "Lower version"
#do_test simple10 "Higher version"
do_test simple11 "Priority (ne)"
do_test simple12 "Priority (eq)"
do_test simple8 "Stickiness"
echo ""
do_test params-0 "Params: No change"
do_test params-1 "Params: Changed"
do_test params-2 "Params: Resource definition"
do_test params-4 "Params: Reload"
do_test novell-251689 "Resource definition change + target_role=stopped"
echo ""
do_test orphan-0 "Orphan ignore"
do_test orphan-1 "Orphan stop"
echo ""
do_test target-0 "Target Role : baseline"
do_test target-1 "Target Role : test"
echo ""
do_test date-1 "Dates" -d "2005-020"
do_test date-2 "Date Spec - Pass" -d "2005-020T12:30"
do_test date-3 "Date Spec - Fail" -d "2005-020T11:30"
do_test probe-0 "Probe (anon clone)"
do_test probe-1 "Pending Probe"
do_test standby "Standby"
do_test comments "Comments"
echo ""
do_test rsc_dep1 "Must not "
do_test rsc_dep3 "Must "
do_test rsc_dep5 "Must not 3 "
do_test rsc_dep7 "Must 3 "
do_test rsc_dep10 "Must (but cant)"
do_test rsc_dep2 "Must (running) "
do_test rsc_dep8 "Must (running : alt) "
do_test rsc_dep4 "Must (running + move)"
echo ""
do_test order1 "Order start 1 "
do_test order2 "Order start 2 "
do_test order3 "Order stop "
do_test order4 "Order (multiple) "
do_test order5 "Order (move) "
do_test order6 "Order (move w/ restart) "
do_test order7 "Order (manditory) "
do_test order-optional "Order (score=0) "
do_test order-required "Order (score=INFINITY) "
echo ""
do_test coloc-loop "Colocation - loop"
do_test coloc-many-one "Colocation - many-to-one"
do_test coloc-list "Colocation - many-to-one with list"
do_test coloc-group "Colocation - groups"
#echo ""
#do_test agent1 "version: lt (empty)"
#do_test agent2 "version: eq "
#do_test agent3 "version: gt "
echo ""
do_test attrs1 "string: eq (and) "
do_test attrs2 "string: lt / gt (and)"
do_test attrs3 "string: ne (or) "
do_test attrs4 "string: exists "
do_test attrs5 "string: not_exists "
do_test attrs6 "is_dc: true "
do_test attrs7 "is_dc: false "
do_test attrs8 "score_attribute "
echo ""
do_test mon-rsc-1 "Schedule Monitor - start"
do_test mon-rsc-2 "Schedule Monitor - move "
do_test mon-rsc-3 "Schedule Monitor - pending start "
do_test mon-rsc-4 "Schedule Monitor - move/pending start"
echo ""
do_test rec-rsc-0 "Resource Recover - no start "
do_test rec-rsc-1 "Resource Recover - start "
do_test rec-rsc-2 "Resource Recover - monitor "
do_test rec-rsc-3 "Resource Recover - stop - ignore"
do_test rec-rsc-4 "Resource Recover - stop - block "
do_test rec-rsc-5 "Resource Recover - stop - fence "
do_test rec-rsc-6 "Resource Recover - multiple - restart"
do_test rec-rsc-7 "Resource Recover - multiple - stop "
do_test rec-rsc-8 "Resource Recover - multiple - block "
do_test rec-rsc-9 "Resource Recover - group/group"
echo ""
do_test quorum-1 "No quorum - ignore"
do_test quorum-2 "No quorum - freeze"
do_test quorum-3 "No quorum - stop "
do_test quorum-4 "No quorum - start anyway"
do_test quorum-5 "No quorum - start anyway (group)"
do_test quorum-6 "No quorum - start anyway (clone)"
echo ""
do_test rec-node-1 "Node Recover - Startup - no fence"
do_test rec-node-2 "Node Recover - Startup - fence "
do_test rec-node-3 "Node Recover - HA down - no fence"
do_test rec-node-4 "Node Recover - HA down - fence "
do_test rec-node-5 "Node Recover - CRM down - no fence"
do_test rec-node-6 "Node Recover - CRM down - fence "
do_test rec-node-7 "Node Recover - no quorum - ignore "
do_test rec-node-8 "Node Recover - no quorum - freeze "
do_test rec-node-9 "Node Recover - no quorum - stop "
do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
do_test rec-node-12 "Node Recover - nothing active - fence "
do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
do_test rec-node-15 "Node Recover - unknown lrm section"
do_test rec-node-14 "Serialize all stonith's"
echo ""
do_test multi1 "Multiple Active (stop/start)"
echo ""
do_test migrate-1 "Migrate (migrate)"
do_test migrate-2 "Migrate (stable)"
do_test migrate-3 "Migrate (failed migrate_to)"
do_test migrate-4 "Migrate (failed migrate_from)"
do_test novell-252693 "Migration in a stopping stack"
do_test novell-252693-2 "Migration in a starting stack"
do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
#echo ""
#do_test complex1 "Complex "
echo ""
do_test group1 "Group "
do_test group2 "Group + Native "
do_test group3 "Group + Group "
do_test group4 "Group + Native (nothing)"
do_test group5 "Group + Native (move) "
do_test group6 "Group + Group (move) "
do_test group7 "Group colocation"
do_test group13 "Group colocation (cant run)"
do_test group8 "Group anti-colocation"
do_test group9 "Group recovery"
do_test group10 "Group partial recovery"
do_test group11 "Group target_role"
do_test group14 "Group stop (graph terminated)"
do_test group15 "-ve group colocation"
do_test bug-1573 "Partial stop of a group with two children"
do_test bug-1718 "Mandatory group ordering - Stop group_FUN"
echo ""
do_test inc0 "Incarnation start"
do_test inc1 "Incarnation start order"
do_test inc2 "Incarnation silent restart, stop, move"
do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
do_test inc7 "Clone colocation"
do_test inc8 "Clone anti-colocation"
do_test inc9 "Non-unique clone"
do_test inc10 "Non-unique clone (stop)"
do_test inc11 "Primitive colocation with clones"
do_test inc12 "Clone shutdown"
do_test cloned-group "Make sure only the correct number of cloned groups are started"
echo ""
do_test master-0 "Stopped -> Slave"
do_test master-1 "Stopped -> Promote"
do_test master-2 "Stopped -> Promote : notify"
do_test master-3 "Stopped -> Promote : master location"
do_test master-4 "Started -> Promote : master location"
do_test master-5 "Promoted -> Promoted"
do_test master-6 "Promoted -> Promoted (2)"
do_test master-7 "Promoted -> Fenced"
do_test master-8 "Promoted -> Fenced -> Moved"
do_test master-9 "Stopped + Promotable + No quorum"
do_test master-10 "Stopped -> Promotable : notify with monitor"
do_test master-11 "Stopped -> Promote : colocation"
do_test novell-239082 "Demote/Promote ordering"
do_test novell-239087 "Stable master placement"
do_test master-12 "Promotion based solely on rsc_location constraints"
do_test master-13 "Include preferences of colocated resources when placing master"
do_test master-demote "Ordering when actions depends on demoting a slave resource"
do_test master-ordering "Prevent resources from starting that need a master"
+do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
echo ""
do_test managed-0 "Managed (reference)"
do_test managed-1 "Not managed - down "
do_test managed-2 "Not managed - up "
echo ""
do_test interleave-0 "Interleave (reference)"
do_test interleave-1 "coloc - not interleaved"
do_test interleave-2 "coloc - interleaved "
do_test interleave-3 "coloc - interleaved (2)"
echo ""
do_test notify-0 "Notify reference"
do_test notify-1 "Notify simple"
do_test notify-2 "Notify simple, confirm"
do_test notify-3 "Notify move, confirm"
do_test novell-239079 "Notification priority"
#do_test notify-2 "Notify - 764"
echo ""
do_test 594 "OSDL #594"
do_test 662 "OSDL #662"
do_test 696 "OSDL #696"
do_test 726 "OSDL #726"
do_test 735 "OSDL #735"
do_test 764 "OSDL #764"
do_test 797 "OSDL #797"
do_test 829 "OSDL #829"
do_test 994 "OSDL #994"
do_test 1360 "OSDL #1360 - Clone stickiness"
do_test 1484 "OSDL #1484 - on_fail=stop"
do_test 1494 "OSDL #1494 - Clone stability"
do_test unrunnable-1 "Unrunnable"
do_test stonith-0 "Stonith loop - 1"
do_test stonith-1 "Stonith loop - 2"
do_test bug-1572-1 "Recovery of groups depending on master/slave"
do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
do_test bug-1685 "Depends-on-master ordering"
echo ""
test_results
diff --git a/crm/pengine/testcases/bug-1765.dot b/crm/pengine/testcases/bug-1765.dot
new file mode 100644
index 0000000000..2b20567e91
--- /dev/null
+++ b/crm/pengine/testcases/bug-1765.dot
@@ -0,0 +1,25 @@
+digraph "g" {
+"drbd0:0_post_notify_start_0 sles236" -> "ms-drbd0_confirmed-post_notify_start_0" [ style = bold]
+"drbd0:0_post_notify_start_0 sles236" [ style=bold color="green" fontcolor="black" ]
+"drbd0:0_pre_notify_start_0 sles236" -> "ms-drbd0_confirmed-pre_notify_start_0" [ style = bold]
+"drbd0:0_pre_notify_start_0 sles236" [ style=bold color="green" fontcolor="black" ]
+"drbd0:1_post_notify_start_0 sles238" -> "ms-drbd0_confirmed-post_notify_start_0" [ style = bold]
+"drbd0:1_post_notify_start_0 sles238" [ style=bold color="green" fontcolor="black" ]
+"drbd0:1_start_0 sles238" -> "ms-drbd0_running_0" [ style = bold]
+"drbd0:1_start_0 sles238" [ style=bold color="green" fontcolor="black" ]
+"ms-drbd0_confirmed-post_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
+"ms-drbd0_confirmed-pre_notify_start_0" -> "ms-drbd0_start_0" [ style = bold]
+"ms-drbd0_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
+"ms-drbd0_post_notify_start_0" -> "drbd0:0_post_notify_start_0 sles236" [ style = bold]
+"ms-drbd0_post_notify_start_0" -> "drbd0:1_post_notify_start_0 sles238" [ style = bold]
+"ms-drbd0_post_notify_start_0" -> "ms-drbd0_confirmed-post_notify_start_0" [ style = bold]
+"ms-drbd0_post_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
+"ms-drbd0_pre_notify_start_0" -> "drbd0:0_pre_notify_start_0 sles236" [ style = bold]
+"ms-drbd0_pre_notify_start_0" -> "ms-drbd0_confirmed-pre_notify_start_0" [ style = bold]
+"ms-drbd0_pre_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
+"ms-drbd0_running_0" -> "ms-drbd0_post_notify_start_0" [ style = bold]
+"ms-drbd0_running_0" [ style=bold color="green" fontcolor="orange" ]
+"ms-drbd0_start_0" -> "drbd0:1_start_0 sles238" [ style = bold]
+"ms-drbd0_start_0" -> "ms-drbd0_running_0" [ style = bold]
+"ms-drbd0_start_0" [ style=bold color="green" fontcolor="orange" ]
+}
diff --git a/crm/pengine/testcases/bug-1765.exp b/crm/pengine/testcases/bug-1765.exp
new file mode 100644
index 0000000000..caf56ebad1
--- /dev/null
+++ b/crm/pengine/testcases/bug-1765.exp
@@ -0,0 +1,135 @@
+ <transition_graph cluster-delay="60s" transition_id="0">
+ <synapse id="0">
+ <action_set>
+ <rsc_op id="73" operation="notify" operation_key="drbd0:0_pre_notify_start_0" on_node="sles236" on_node_uuid="aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <primitive id="drbd0:0" long-id="ms-drbd0:drbd0:0" class="ocf" provider="heartbeat" type="drbd"/>
+ <attributes crm_feature_set="2.0" CRM_meta_notify_operation="start" CRM_meta_timeout="20000" CRM_meta_notify_confirm="yes" CRM_meta_notify_type="pre" CRM_meta_role="Master" drbd_resource="tr2" CRM_meta_stateful="true" CRM_meta_notify_master_uname="sles236 " CRM_meta_notify_start_uname="sles238 " CRM_meta_notify_master_resource="drbd0:0 " CRM_meta_notify_start_resource="drbd0:1 " CRM_meta_globally_unique="false" CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="15" operation="notify" operation_key="ms-drbd0_pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="1" priority="1000000">
+ <action_set>
+ <rsc_op id="74" operation="notify" operation_key="drbd0:0_post_notify_start_0" on_node="sles236" on_node_uuid="aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <primitive id="drbd0:0" long-id="ms-drbd0:drbd0:0" class="ocf" provider="heartbeat" type="drbd"/>
+ <attributes crm_feature_set="2.0" CRM_meta_notify_operation="start" CRM_meta_timeout="20000" CRM_meta_notify_confirm="yes" CRM_meta_notify_type="post" CRM_meta_role="Master" drbd_resource="tr2" CRM_meta_stateful="true" CRM_meta_notify_master_uname="sles236 " CRM_meta_notify_start_uname="sles238 " CRM_meta_notify_master_resource="drbd0:0 " CRM_meta_notify_start_resource="drbd0:1 " CRM_meta_globally_unique="false" CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="17" operation="notify" operation_key="ms-drbd0_post_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+ <action_set>
+ <rsc_op id="12" operation="start" operation_key="drbd0:1_start_0" on_node="sles238" on_node_uuid="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <primitive id="drbd0:1" long-id="ms-drbd0:drbd0:1" class="ocf" provider="heartbeat" type="drbd"/>
+ <attributes crm_feature_set="2.0" drbd_resource="tr2" CRM_meta_stateful="true" CRM_meta_notify_master_uname="sles236 " CRM_meta_timeout="20000" CRM_meta_notify_start_uname="sles238 " CRM_meta_notify_master_resource="drbd0:0 " CRM_meta_notify_start_resource="drbd0:1 " CRM_meta_globally_unique="false" CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="13" operation="start" operation_key="ms-drbd0_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="3" priority="1000000">
+ <action_set>
+ <rsc_op id="81" operation="notify" operation_key="drbd0:1_post_notify_start_0" on_node="sles238" on_node_uuid="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <primitive id="drbd0:1" long-id="ms-drbd0:drbd0:1" class="ocf" provider="heartbeat" type="drbd"/>
+ <attributes crm_feature_set="2.0" CRM_meta_notify_operation="start" CRM_meta_timeout="20000" CRM_meta_notify_confirm="yes" CRM_meta_notify_type="post" drbd_resource="tr2" CRM_meta_stateful="true" CRM_meta_notify_master_uname="sles236 " CRM_meta_notify_start_uname="sles238 " CRM_meta_notify_master_resource="drbd0:0 " CRM_meta_notify_start_resource="drbd0:1 " CRM_meta_globally_unique="false" CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </rsc_op>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="17" operation="notify" operation_key="ms-drbd0_post_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="4">
+ <action_set>
+ <pseudo_event id="13" operation="start" operation_key="ms-drbd0_start_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="16" operation="notified" operation_key="ms-drbd0_confirmed-pre_notify_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5" priority="1000000">
+ <action_set>
+ <pseudo_event id="14" operation="running" operation_key="ms-drbd0_running_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <rsc_op id="12" operation="start" operation_key="drbd0:1_start_0" on_node="sles238" on_node_uuid="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78"/>
+ </trigger>
+ <trigger>
+ <pseudo_event id="13" operation="start" operation_key="ms-drbd0_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+ <action_set>
+ <pseudo_event id="15" operation="notify" operation_key="ms-drbd0_pre_notify_start_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_notify_confirm="yes" CRM_meta_notify_type="pre" CRM_meta_notify_operation="start" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs/>
+ </synapse>
+ <synapse id="7">
+ <action_set>
+ <pseudo_event id="16" operation="notified" operation_key="ms-drbd0_confirmed-pre_notify_start_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_notify_type="pre" CRM_meta_notify_operation="start" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="15" operation="notify" operation_key="ms-drbd0_pre_notify_start_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="73" operation="notify" operation_key="drbd0:0_pre_notify_start_0" on_node="sles236" on_node_uuid="aa584ceb-0d17-48ed-97c2-250b062e7407"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="8" priority="1000000">
+ <action_set>
+ <pseudo_event id="17" operation="notify" operation_key="ms-drbd0_post_notify_start_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_notify_confirm="yes" CRM_meta_notify_type="post" CRM_meta_notify_operation="start" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="14" operation="running" operation_key="ms-drbd0_running_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="9" priority="1000000">
+ <action_set>
+ <pseudo_event id="18" operation="notified" operation_key="ms-drbd0_confirmed-post_notify_start_0">
+ <attributes crm_feature_set="2.0" CRM_meta_stateful="true" CRM_meta_timeout="20000" CRM_meta_notify_type="pre" CRM_meta_notify_operation="start" CRM_meta_globally_unique="false" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_master_max="1" CRM_meta_master_node_max="1"/>
+ </pseudo_event>
+ </action_set>
+ <inputs>
+ <trigger>
+ <pseudo_event id="17" operation="notify" operation_key="ms-drbd0_post_notify_start_0"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="74" operation="notify" operation_key="drbd0:0_post_notify_start_0" on_node="sles236" on_node_uuid="aa584ceb-0d17-48ed-97c2-250b062e7407"/>
+ </trigger>
+ <trigger>
+ <rsc_op id="81" operation="notify" operation_key="drbd0:1_post_notify_start_0" on_node="sles238" on_node_uuid="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ </transition_graph>
+
diff --git a/crm/pengine/testcases/bug-1765.xml b/crm/pengine/testcases/bug-1765.xml
new file mode 100644
index 0000000000..6a335e0f7b
--- /dev/null
+++ b/crm/pengine/testcases/bug-1765.xml
@@ -0,0 +1,216 @@
+ <cib generated="true" admin_epoch="0" have_quorum="true" ignore_dtd="false" num_peers="2" ccm_transition="2" cib_feature_revision="2.0" dc_uuid="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78" epoch="61" num_updates="26">
+ <configuration>
+ <crm_config>
+ <cluster_property_set id="cib-bootstrap-options">
+ <attributes>
+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="2.1.2-node: 394e7c3241c59ee0e538643fde4b60cddaf5d3f4"/>
+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1193843111"/>
+ </attributes>
+ </cluster_property_set>
+ </crm_config>
+ <nodes>
+ <node id="aa584ceb-0d17-48ed-97c2-250b062e7407" uname="sles236" type="normal">
+ <instance_attributes id="nodes-aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <attributes>
+ <nvpair id="standby-aa584ceb-0d17-48ed-97c2-250b062e7407" name="standby" value="off"/>
+ </attributes>
+ </instance_attributes>
+ </node>
+ <node uname="sles238" type="normal" id="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <instance_attributes id="nodes-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <attributes>
+ <nvpair name="standby" id="standby-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78" value="off"/>
+ </attributes>
+ </instance_attributes>
+ </node>
+ </nodes>
+ <resources>
+ <master_slave id="ms-drbd0">
+ <meta_attributes id="ma-ms-drbd0">
+ <attributes>
+ <nvpair id="ma-ms-drbd0-1" name="clone_max" value="2"/>
+ <nvpair id="ma-ms-drbd0-2" name="clone_node_max" value="1"/>
+ <nvpair id="ma-ms-drbd0-3" name="master_max" value="1"/>
+ <nvpair id="ma-ms-drbd0-4" name="master_node_max" value="1"/>
+ <nvpair id="ma-ms-drbd0-5" name="notify" value="yes"/>
+ <nvpair id="ma-ms-drbd0-6" name="globally_unique" value="false"/>
+ <nvpair id="ma-ms-drbd0-8" name="interleave" value="true"/>
+ <nvpair id="ma-ms-drbd0-7" name="target_role" value="started"/>
+ </attributes>
+ </meta_attributes>
+ <primitive id="drbd0" class="ocf" provider="heartbeat" type="drbd">
+ <instance_attributes id="ia-drbd0">
+ <attributes>
+ <nvpair id="ia-drbd0-1" name="drbd_resource" value="tr2"/>
+ </attributes>
+ </instance_attributes>
+ <operations>
+ <op id="drbd0_mon_0" name="monitor" interval="12s" timeout="5s" start_delay="0" disabled="false" role="Started"/>
+ <op id="drbd0_mon_1" name="monitor" interval="10s" timeout="5s" role="Master" start_delay="0" disabled="false"/>
+ </operations>
+ <meta_attributes id="drbd0:1_meta_attrs">
+ <attributes>
+ <nvpair id="drbd0:1_metaattr_target_role" name="target_role" value="started"/>
+ </attributes>
+ </meta_attributes>
+ </primitive>
+ </master_slave>
+ <master_slave id="ms-drbd1">
+ <meta_attributes id="ma-ms-drbd1">
+ <attributes>
+ <nvpair id="ma-ms-drbd1-1" name="clone_max" value="2"/>
+ <nvpair id="ma-ms-drbd1-2" name="clone_node_max" value="1"/>
+ <nvpair id="ma-ms-drbd1-3" name="master_max" value="1"/>
+ <nvpair id="ma-ms-drbd1-4" name="master_node_max" value="1"/>
+ <nvpair id="ma-ms-drbd1-5" name="notify" value="yes"/>
+ <nvpair id="ma-ms-drbd1-6" name="globally_unique" value="false"/>
+ <nvpair id="ma-ms-drbd1-7" name="target_role" value="started"/>
+ <nvpair id="ma-ms-drbd1-8" name="interleave" value="true"/>
+ </attributes>
+ </meta_attributes>
+ <primitive id="drbd1" class="ocf" provider="heartbeat" type="drbd">
+ <instance_attributes id="ia-drbd1">
+ <attributes>
+ <nvpair id="ia-drbd1-1" name="drbd_resource" value="tr2-var"/>
+ </attributes>
+ </instance_attributes>
+ <operations>
+ <op id="drbd1_mon_0" name="monitor" interval="12s" timeout="5s"/>
+ <op id="drbd1_mon_1" name="monitor" interval="10s" timeout="5s" role="Master"/>
+ </operations>
+ <meta_attributes id="drbd1:0_meta_attrs">
+ <attributes>
+ <nvpair id="drbd1:0_metaattr_target_role" name="target_role" value="started"/>
+ </attributes>
+ </meta_attributes>
+ </primitive>
+ </master_slave>
+ </resources>
+ <constraints>
+ <rsc_colocation id="drbd0_drbd1_master" from="ms-drbd0" from_role="master" to="ms-drbd1" to_role="master" score="INFINITY"/>
+ </constraints>
+ </configuration>
+ <status>
+ <node_state uname="sles238" crmd="online" shutdown="0" in_ccm="true" ha="active" join="member" expected="member" id="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <transient_attributes id="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <instance_attributes id="status-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <attributes>
+ <nvpair id="status-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78-probe_complete" name="probe_complete" value="true"/>
+ <nvpair id="status-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78-fail-count-drbd0:0" name="fail-count-drbd0:0" value="2"/>
+ <nvpair id="status-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78-fail-count-drbd0:1" name="fail-count-drbd0:1" value="1"/>
+ </attributes>
+ </instance_attributes>
+ <instance_attributes id="master-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <attributes>
+ <nvpair id="status-master-drbd0:1-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78" name="master-drbd0:1" value="5"/>
+ <nvpair name="master-drbd1:1" id="status-master-drbd1:1-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78" value="75"/>
+ <nvpair name="master-drbd0:0" id="status-master-drbd0:0-f1c6f9c2-3e78-4ac4-b77c-215b4457fc78" value="75"/>
+ </attributes>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="f1c6f9c2-3e78-4ac4-b77c-215b4457fc78">
+ <lrm_resources>
+ <lrm_resource type="drbd" class="ocf" provider="heartbeat" id="drbd1:1">
+ <lrm_rsc_op id="drbd1:1_promote_0" operation="promote" crm-debug-origin="build_active_RAs" transition_key="39:33:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;39:33:3282115e-0c7f-4f4e-925f-94870134713a" call_id="102" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:1_demote_0" operation="demote" crm-debug-origin="build_active_RAs" transition_key="45:36:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;45:36:3282115e-0c7f-4f4e-925f-94870134713a" call_id="114" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op operation="stop" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" id="drbd1:1_stop_0" transition_key="43:128:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;43:128:3282115e-0c7f-4f4e-925f-94870134713a" call_id="239" op_digest="45c27a3fb903b07d748f5d7809222ed4"/>
+ <lrm_rsc_op operation="start" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac" id="drbd1:1_start_0" transition_key="37:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;37:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="247"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac" id="drbd1:1_post_notify_promote_0" transition_key="102:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;102:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="253"/>
+ <lrm_rsc_op id="drbd1:1_notify_0" operation="notify" crm_feature_set="2.0" op_status="-1" call_id="-1" interval="0" rc_code="-1" crm-debug-origin="cib_action_update" transition_key="92:126:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="-1:-1;92:126:3282115e-0c7f-4f4e-925f-94870134713a" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:1_pre_notify_stop_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="93:128:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;93:128:3282115e-0c7f-4f4e-925f-94870134713a" call_id="238" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="45c27a3fb903b07d748f5d7809222ed4"/>
+ <lrm_rsc_op id="drbd1:1_post_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="86:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;86:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="249" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:1_pre_notify_promote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="101:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;101:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="251" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:1_monitor_12000" operation="monitor" crm-debug-origin="do_update_resource" transition_key="45:134:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;45:134:3282115e-0c7f-4f4e-925f-94870134713a" call_id="254" crm_feature_set="2.0" rc_code="0" op_status="0" interval="12000" op_digest="001ce1a2346263e1fb90f626ba0799a1"/>
+ </lrm_resource>
+ <lrm_resource type="drbd" class="ocf" provider="heartbeat" id="drbd0:0">
+ <lrm_rsc_op operation="start" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_start_0" transition_key="7:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;7:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="246"/>
+ <lrm_rsc_op id="drbd0:0_promote_0" operation="promote" crm-debug-origin="build_active_RAs" transition_key="10:119:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;10:119:3282115e-0c7f-4f4e-925f-94870134713a" call_id="231" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op id="drbd0:0_pre_notify_start_0" operation="notify" crm-debug-origin="build_active_RAs" transition_key="75:122:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;75:122:3282115e-0c7f-4f4e-925f-94870134713a" call_id="234" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_pre_notify_demote_0" transition_key="78:136:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;78:136:3282115e-0c7f-4f4e-925f-94870134713a" call_id="255"/>
+ <lrm_rsc_op id="drbd0:0_demote_0" operation="demote" crm-debug-origin="do_update_resource" transition_key="13:128:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;13:128:3282115e-0c7f-4f4e-925f-94870134713a" call_id="240" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_post_notify_demote_0" transition_key="79:136:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;79:136:3282115e-0c7f-4f4e-925f-94870134713a" call_id="256"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_pre_notify_stop_0" transition_key="75:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;75:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="258"/>
+ <lrm_rsc_op operation="stop" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_stop_0" transition_key="10:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;10:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="259"/>
+ <lrm_rsc_op id="drbd0:0_post_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="75:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;75:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="248" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op id="drbd0:0_pre_notify_promote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="85:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;85:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="250" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op id="drbd0:0_post_notify_promote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="86:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;86:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="252" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ </lrm_resource>
+ <lrm_resource id="drbd1:0" type="drbd" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="drbd1:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" transition_key="7:129:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:7;7:129:3282115e-0c7f-4f4e-925f-94870134713a" call_id="242" crm_feature_set="2.0" rc_code="7" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ </lrm_resource>
+ <lrm_resource id="drbd0:1" type="drbd" class="ocf" provider="heartbeat">
+ <lrm_rsc_op id="drbd0:1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" transition_key="5:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:7;5:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="245" crm_feature_set="2.0" rc_code="7" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ <node_state uname="sles236" crmd="online" in_ccm="true" ha="active" join="member" expected="member" shutdown="0" id="aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <transient_attributes id="aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <instance_attributes id="status-aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <attributes>
+ <nvpair id="status-aa584ceb-0d17-48ed-97c2-250b062e7407-probe_complete" name="probe_complete" value="true"/>
+ <nvpair id="status-aa584ceb-0d17-48ed-97c2-250b062e7407-fail-count-drbd1:0" name="fail-count-drbd1:0" value="2"/>
+ <nvpair id="status-aa584ceb-0d17-48ed-97c2-250b062e7407-fail-count-drbd0:0" name="fail-count-drbd0:0" value="2"/>
+ <nvpair id="status-aa584ceb-0d17-48ed-97c2-250b062e7407-fail-count-drbd1:1" name="fail-count-drbd1:1" value="2"/>
+ <nvpair id="status-aa584ceb-0d17-48ed-97c2-250b062e7407-fail-count-drbd0:1" name="fail-count-drbd0:1" value="3"/>
+ </attributes>
+ </instance_attributes>
+ <instance_attributes id="master-aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <attributes>
+ <nvpair name="master-drbd0:0" id="status-master-drbd0:0-aa584ceb-0d17-48ed-97c2-250b062e7407" value="10"/>
+ <nvpair name="master-drbd1:0" id="status-master-drbd1:0-aa584ceb-0d17-48ed-97c2-250b062e7407" value="75"/>
+ <nvpair name="master-drbd0:1" id="status-master-drbd0:1-aa584ceb-0d17-48ed-97c2-250b062e7407" value="75"/>
+ <nvpair id="status-master-drbd1:1-aa584ceb-0d17-48ed-97c2-250b062e7407" name="master-drbd1:1" value="10"/>
+ </attributes>
+ </instance_attributes>
+ </transient_attributes>
+ <lrm id="aa584ceb-0d17-48ed-97c2-250b062e7407">
+ <lrm_resources>
+ <lrm_resource type="drbd" class="ocf" provider="heartbeat" id="drbd1:0">
+ <lrm_rsc_op id="drbd1:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" transition_key="5:126:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:8;5:126:3282115e-0c7f-4f4e-925f-94870134713a" call_id="261" crm_feature_set="2.0" rc_code="8" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_pre_notify_stop_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="87:128:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;87:128:3282115e-0c7f-4f4e-925f-94870134713a" call_id="265" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_post_notify_stop_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="88:128:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;88:128:3282115e-0c7f-4f4e-925f-94870134713a" call_id="267" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_pre_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="89:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;89:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="286" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_pre_notify_demote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="95:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;95:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="287" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_demote_0" operation="demote" crm-debug-origin="do_update_resource" transition_key="42:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;42:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="289" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_post_notify_demote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="96:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;96:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="291" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_post_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="90:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;90:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="293" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_pre_notify_promote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="93:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;93:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="295" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_promote_0" operation="promote" crm-debug-origin="do_update_resource" transition_key="40:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;40:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="297" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_post_notify_promote_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="94:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;94:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="299" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="4951be763ef4103d2ecb25a71430ccac"/>
+ <lrm_rsc_op id="drbd1:0_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" transition_key="41:134:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:8;41:134:3282115e-0c7f-4f4e-925f-94870134713a" call_id="301" crm_feature_set="2.0" rc_code="8" op_status="0" interval="10000" op_digest="001ce1a2346263e1fb90f626ba0799a1"/>
+ </lrm_resource>
+ <lrm_resource type="drbd" class="ocf" provider="heartbeat" id="drbd0:0">
+ <lrm_rsc_op operation="demote" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_demote_0" transition_key="10:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;10:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="274"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_pre_notify_stop_0" transition_key="76:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;76:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="276"/>
+ <lrm_rsc_op operation="stop" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_stop_0" transition_key="11:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;11:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="277"/>
+ <lrm_rsc_op id="drbd0:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="6:122:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;6:122:3282115e-0c7f-4f4e-925f-94870134713a" call_id="260" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_pre_notify_promote_0" transition_key="76:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;76:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="310"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_pre_notify_demote_0" transition_key="77:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;77:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="273"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_post_notify_demote_0" transition_key="78:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;78:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="275"/>
+ <lrm_rsc_op operation="promote" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_promote_0" transition_key="12:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;12:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="311"/>
+ <lrm_rsc_op id="drbd0:0_post_notify_stop_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="77:129:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;77:129:3282115e-0c7f-4f4e-925f-94870134713a" call_id="271" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:0_post_notify_promote_0" transition_key="77:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;77:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="312"/>
+ <lrm_rsc_op id="drbd0:0_start_0" operation="start" crm-debug-origin="do_update_resource" transition_key="11:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;11:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="308" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op id="drbd0:0_post_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="74:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;74:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="309" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op id="drbd0:0_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" transition_key="13:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:8;13:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="313" crm_feature_set="2.0" rc_code="8" op_status="0" interval="10000" op_digest="841c5dd8e61e623af01d5b7f0ec80a2c"/>
+ </lrm_resource>
+ <lrm_resource type="drbd" class="ocf" provider="heartbeat" id="drbd0:1">
+ <lrm_rsc_op id="drbd0:1_start_0" operation="start" crm-debug-origin="do_update_resource" transition_key="7:130:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;7:130:3282115e-0c7f-4f4e-925f-94870134713a" call_id="278" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_post_notify_start_0" transition_key="79:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;79:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="292"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_pre_notify_promote_0" transition_key="77:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;77:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="294"/>
+ <lrm_rsc_op operation="promote" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_promote_0" transition_key="8:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;8:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="296"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_post_notify_promote_0" transition_key="78:133:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;78:133:3282115e-0c7f-4f4e-925f-94870134713a" call_id="298"/>
+ <lrm_rsc_op id="drbd0:1_pre_notify_start_0" operation="notify" crm-debug-origin="do_update_resource" transition_key="78:132:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;78:132:3282115e-0c7f-4f4e-925f-94870134713a" call_id="284" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_pre_notify_demote_0" transition_key="81:136:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;81:136:3282115e-0c7f-4f4e-925f-94870134713a" call_id="302"/>
+ <lrm_rsc_op operation="demote" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_demote_0" transition_key="13:136:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;13:136:3282115e-0c7f-4f4e-925f-94870134713a" call_id="303"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_post_notify_demote_0" transition_key="82:136:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;82:136:3282115e-0c7f-4f4e-925f-94870134713a" call_id="304"/>
+ <lrm_rsc_op operation="notify" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31" id="drbd0:1_pre_notify_stop_0" transition_key="80:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;80:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="306"/>
+ <lrm_rsc_op id="drbd0:1_stop_0" operation="stop" crm-debug-origin="do_update_resource" transition_key="4:137:3282115e-0c7f-4f4e-925f-94870134713a" transition_magic="0:0;4:137:3282115e-0c7f-4f4e-925f-94870134713a" call_id="307" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="de2c06ba097c43235d6dca55f93d4f31"/>
+ </lrm_resource>
+ </lrm_resources>
+ </lrm>
+ </node_state>
+ </status>
+ </cib>
+