No OneTemporary
Actions

Size

93 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/pengine/clone.c b/pengine/clone.c
	index 5459f4d2c5..ec9c32eafa 100644
	--- a/pengine/clone.c
	+++ b/pengine/clone.c
	@@ -1,1689 +1,1699 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>
	#include <lib/pengine/utils.h>

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	gint sort_clone_instance(gconstpointer a, gconstpointer b);

	void child_stopping_constraints(
	clone_variant_data_t *clone_data,
	resource_t self, resource_t child, resource_t *last,
	pe_working_set_t *data_set);

	void child_starting_constraints(
	clone_variant_data_t *clone_data,
	resource_t self, resource_t child, resource_t *last,
	pe_working_set_t *data_set);

	static node_t *
	parent_node_instance(const resource_t rsc, node_t node)
	{
	node_t *ret = NULL;
	if(node != NULL) {
	ret = pe_find_node_id(
	rsc->parent->allowed_nodes, node->details->id);
	}
	return ret;
	}

	static gboolean did_fail(const resource_t *rsc)
	{
	if(is_set(rsc->flags, pe_rsc_failed)) {
	return TRUE;
	}

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	if(did_fail(child_rsc)) {
	return TRUE;
	}
	);
	return FALSE;
	}


	gint sort_clone_instance(gconstpointer a, gconstpointer b)
	{
	int level = LOG_DEBUG_3;
	node_t *node1 = NULL;
	node_t *node2 = NULL;

	gboolean can1 = TRUE;
	gboolean can2 = TRUE;
	gboolean with_scores = TRUE;

	const resource_t resource1 = (const resource_t)a;
	const resource_t resource2 = (const resource_t)b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	/* allocation order:
	* - active instances
	* - instances running on nodes with the least copies
	* - active instances on nodes that cant support them or are to be fenced
	* - failed instances
	* - inactive instances
	*/

	do_crm_log_unlikely(level+1, "%s ? %s", resource1->id, resource2->id);
	if(resource1->running_on && resource2->running_on) {
	if(g_list_length(resource1->running_on) < g_list_length(resource2->running_on)) {
	do_crm_log_unlikely(level, "%s < %s: running_on", resource1->id, resource2->id);
	return -1;

	} else if(g_list_length(resource1->running_on) > g_list_length(resource2->running_on)) {
	do_crm_log_unlikely(level, "%s > %s: running_on", resource1->id, resource2->id);
	return 1;
	}
	}

	if(resource1->running_on) {
	node1 = resource1->running_on->data;
	}
	if(resource2->running_on) {
	node2 = resource2->running_on->data;
	}

	if(node1) {
	node_t *match = pe_find_node_id(resource1->allowed_nodes, node1->details->id);
	if(match == NULL \|\| match->weight < 0) {
	do_crm_log_unlikely(level, "%s: current location is unavailable", resource1->id);
	node1 = NULL;
	can1 = FALSE;
	}
	}

	if(node2) {
	node_t *match = pe_find_node_id(resource2->allowed_nodes, node2->details->id);
	if(match == NULL \|\| match->weight < 0) {
	do_crm_log_unlikely(level, "%s: current location is unavailable", resource2->id);
	node2 = NULL;
	can2 = FALSE;
	}
	}

	if(can1 != can2) {
	if(can1) {
	do_crm_log_unlikely(level, "%s < %s: availability of current location", resource1->id, resource2->id);
	return -1;
	}
	do_crm_log_unlikely(level, "%s > %s: availability of current location", resource1->id, resource2->id);
	return 1;
	}

	if(resource1->priority < resource2->priority) {
	do_crm_log_unlikely(level, "%s < %s: priority", resource1->id, resource2->id);
	return 1;

	} else if(resource1->priority > resource2->priority) {
	do_crm_log_unlikely(level, "%s > %s: priority", resource1->id, resource2->id);
	return -1;
	}

	if(node1 == NULL && node2 == NULL) {
	do_crm_log_unlikely(level, "%s == %s: not active",
	resource1->id, resource2->id);
	return 0;
	}

	if(node1 != node2) {
	if(node1 == NULL) {
	do_crm_log_unlikely(level, "%s > %s: active", resource1->id, resource2->id);
	return 1;
	} else if(node2 == NULL) {
	do_crm_log_unlikely(level, "%s < %s: active", resource1->id, resource2->id);
	return -1;
	}
	}

	can1 = can_run_resources(node1);
	can2 = can_run_resources(node2);
	if(can1 != can2) {
	if(can1) {
	do_crm_log_unlikely(level, "%s < %s: can", resource1->id, resource2->id);
	return -1;
	}
	do_crm_log_unlikely(level, "%s > %s: can", resource1->id, resource2->id);
	return 1;
	}

	node1 = parent_node_instance(resource1, node1);
	node2 = parent_node_instance(resource2, node2);
	if(node1 != NULL && node2 == NULL) {
	do_crm_log_unlikely(level, "%s < %s: not allowed", resource1->id, resource2->id);
	return -1;
	} else if(node1 == NULL && node2 != NULL) {
	do_crm_log_unlikely(level, "%s > %s: not allowed", resource1->id, resource2->id);
	return 1;
	}

	if(node1 == NULL) {
	do_crm_log_unlikely(level, "%s == %s: not allowed", resource1->id, resource2->id);
	return 0;
	}

	if(node1->count < node2->count) {
	do_crm_log_unlikely(level, "%s < %s: count", resource1->id, resource2->id);
	return -1;

	} else if(node1->count > node2->count) {
	do_crm_log_unlikely(level, "%s > %s: count", resource1->id, resource2->id);
	return 1;
	}

	if(with_scores) {
	int max = 0;
	int lpc = 0;
	GListPtr list1 = node_list_dup(resource1->allowed_nodes, FALSE, FALSE);
	GListPtr list2 = node_list_dup(resource2->allowed_nodes, FALSE, FALSE);

	list1 = g_list_sort(list1, sort_node_weight);
	list2 = g_list_sort(list2, sort_node_weight);
	max = g_list_length(list1);
	if(max < g_list_length(list2)) {
	max = g_list_length(list2);
	}

	for(;lpc < max; lpc++) {
	node1 = g_list_nth_data(list1, lpc);
	node2 = g_list_nth_data(list2, lpc);
	if(node1 == NULL) {
	do_crm_log_unlikely(level, "%s < %s: node score NULL", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return 1;
	} else if(node2 == NULL) {
	do_crm_log_unlikely(level, "%s > %s: node score NULL", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return -1;
	}

	if(node1->weight < node2->weight) {
	do_crm_log_unlikely(level, "%s < %s: node score", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return 1;

	} else if(node1->weight > node2->weight) {
	do_crm_log_unlikely(level, "%s > %s: node score", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return -1;
	}
	}

	pe_free_shallow(list1); pe_free_shallow(list2);
	}

	can1 = did_fail(resource1);
	can2 = did_fail(resource2);
	if(can1 != can2) {
	if(can1) {
	do_crm_log_unlikely(level, "%s > %s: failed", resource1->id, resource2->id);
	return 1;
	}
	do_crm_log_unlikely(level, "%s < %s: failed", resource1->id, resource2->id);
	return -1;
	}

	if(node1 && node2) {
	int max = 0;
	int lpc = 0;
	GListPtr list1 = g_list_append(NULL, node_copy(resource1->running_on->data));
	GListPtr list2 = g_list_append(NULL, node_copy(resource2->running_on->data));

	/* Possibly a replacement for the with_scores block above */

	slist_iter(
	constraint, rsc_colocation_t, resource1->parent->rsc_cons_lhs, lpc,
	do_crm_log_unlikely(level+1, "Applying %s to %s", constraint->id, resource1->id);

	list1 = native_merge_weights(
	constraint->rsc_lh, resource1->id, list1,
	constraint->node_attribute,
	constraint->score/INFINITY, FALSE);
	);

	slist_iter(
	constraint, rsc_colocation_t, resource2->parent->rsc_cons_lhs, lpc,
	do_crm_log_unlikely(level+1, "Applying %s to %s", constraint->id, resource2->id);

	list2 = native_merge_weights(
	constraint->rsc_lh, resource2->id, list2,
	constraint->node_attribute,
	constraint->score/INFINITY, FALSE);
	);

	list1 = g_list_sort(list1, sort_node_weight);
	list2 = g_list_sort(list2, sort_node_weight);
	max = g_list_length(list1);
	if(max < g_list_length(list2)) {
	max = g_list_length(list2);
	}

	for(;lpc < max; lpc++) {
	node1 = g_list_nth_data(list1, lpc);
	node2 = g_list_nth_data(list2, lpc);
	if(node1 == NULL) {
	do_crm_log_unlikely(level, "%s < %s: colocated score NULL", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return 1;
	} else if(node2 == NULL) {
	do_crm_log_unlikely(level, "%s > %s: colocated score NULL", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return -1;
	}

	if(node1->weight < node2->weight) {
	do_crm_log_unlikely(level, "%s < %s: colocated score", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return 1;

	} else if(node1->weight > node2->weight) {
	do_crm_log_unlikely(level, "%s > %s: colocated score", resource1->id, resource2->id);
	pe_free_shallow(list1); pe_free_shallow(list2);
	return -1;
	}
	}

	pe_free_shallow(list1); pe_free_shallow(list2);
	}


	do_crm_log_unlikely(level, "%s == %s: default %d", resource1->id, resource2->id, node2->weight);
	return 0;
	}

	static node_t *
	can_run_instance(resource_t rsc, node_t node)
	{
	node_t *local_node = NULL;
	clone_variant_data_t *clone_data = NULL;
	if(can_run_resources(node) == FALSE) {
	goto bail;

	} else if(is_set(rsc->flags, pe_rsc_orphan)) {
	goto bail;
	}

	local_node = parent_node_instance(rsc, node);
	get_clone_variant_data(clone_data, rsc->parent);

	if(local_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed",
	rsc->id, node->details->uname);
	goto bail;

	} else if(local_node->count < clone_data->clone_node_max) {
	return local_node;

	} else {
	crm_debug_2("%s cannot run on %s: node full",
	rsc->id, node->details->uname);
	}

	bail:
	if(node) {
	common_update_score(rsc, node->details->id, -INFINITY);
	}
	return NULL;
	}


	static node_t *
	color_instance(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *chosen = NULL;
	node_t *local_node = NULL;

	crm_debug_2("Processing %s", rsc->id);

	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->fns->location(rsc, NULL, FALSE);

	} else if(is_set(rsc->flags, pe_rsc_allocating)) {
	crm_debug("Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	if(rsc->allowed_nodes) {
	slist_iter(try_node, node_t, rsc->allowed_nodes, lpc,
	can_run_instance(rsc, try_node);
	);
	}

	chosen = rsc->cmds->color(rsc, data_set);
	if(chosen) {
	local_node = pe_find_node_id(
	rsc->parent->allowed_nodes, chosen->details->id);

	if(local_node) {
	local_node->count++;
	} else if(is_set(rsc->flags, pe_rsc_managed)) {
	/* what to do? we can't enforce per-node limits in this case */
	crm_config_err("%s not found in %s (list=%d)",
	chosen->details->id, rsc->parent->id,
	g_list_length(rsc->parent->allowed_nodes));
	}
	}

	return chosen;
	}

	static void append_parent_colocation(resource_t rsc, resource_t child, gboolean all)
	{
	slist_iter(cons, rsc_colocation_t, rsc->rsc_cons, lpc,
	if(all \|\| cons->score < 0 \|\| cons->score == INFINITY) {
	child->rsc_cons = g_list_append(child->rsc_cons, cons);
	}

	);
	slist_iter(cons, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
	if(all \|\| cons->score < 0) {
	child->rsc_cons_lhs = g_list_append(child->rsc_cons_lhs, cons);
	}
	);
	}

	node_t *
	clone_color(resource_t rsc, pe_working_set_t data_set)
	{
	int allocated = 0;
	int available_nodes = 0;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
	return NULL;

	} else if(is_set(rsc->flags, pe_rsc_allocating)) {
	crm_debug("Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	crm_debug_2("Processing %s", rsc->id);

	/* this information is used by sort_clone_instance() when deciding in which
	* order to allocate clone instances
	*/
	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,

	rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
	constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute, constraint->score/INFINITY, TRUE);
	);

	dump_node_scores(show_scores?0:scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes);

	/* count now tracks the number of clones currently allocated */
	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
	node->count = 0;
	);

	slist_iter(child, resource_t, rsc->children, lpc,
	if(g_list_length(child->running_on) > 0) {
	node_t *child_node = child->running_on->data;
	node_t *local_node = parent_node_instance(
	child, child->running_on->data);
	if(local_node) {
	local_node->count++;
	} else {
	crm_err("%s is running on %s which isn't allowed",
	child->id, child_node->details->uname);
	}
	}
	);

	rsc->children = g_list_sort(rsc->children, sort_clone_instance);

	/* count now tracks the number of clones we have allocated */
	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
	node->count = 0;
	);

	rsc->allowed_nodes = g_list_sort(
	rsc->allowed_nodes, sort_node_weight);

	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
	if(can_run_resources(node)) {
	available_nodes++;
	}
	);

	slist_iter(child, resource_t, rsc->children, lpc,
	if(allocated >= clone_data->clone_max) {
	crm_debug("Child %s not allocated - limit reached", child->id);
	resource_location(child, NULL, -INFINITY, "clone_color:limit_reached", data_set);

	} else if (clone_data->clone_max < available_nodes) {
	/* Only include positive colocation preferences of dependant resources
	* if not every node will get a copy of the clone
	*/
	append_parent_colocation(rsc, child, TRUE);

	} else {
	append_parent_colocation(rsc, child, FALSE);
	}

	if(color_instance(child, data_set)) {
	allocated++;
	}
	);

	crm_debug("Allocated %d %s instances of a possible %d",
	allocated, rsc->id, clone_data->clone_max);

	clear_bit(rsc->flags, pe_rsc_provisional);
	clear_bit(rsc->flags, pe_rsc_allocating);

	return NULL;
	}

	static void
	clone_update_pseudo_status(
	resource_t rsc, gboolean stopping, gboolean starting, gboolean active)
	{
	if(rsc->children) {
	slist_iter(child, resource_t, rsc->children, lpc,
	clone_update_pseudo_status(child, stopping, starting, active)
	);
	return;
	}

	CRM_ASSERT(active != NULL);
	CRM_ASSERT(starting != NULL);
	CRM_ASSERT(stopping != NULL);

	if(rsc->running_on) {
	*active = TRUE;
	}

	slist_iter(
	action, action_t, rsc->actions, lpc,

	if(starting && stopping) {
	return;

	} else if(action->optional) {
	crm_debug_3("Skipping optional: %s", action->uuid);
	continue;

	} else if(action->pseudo == FALSE && action->runnable == FALSE){
	crm_debug_3("Skipping unrunnable: %s", action->uuid);
	continue;

	} else if(safe_str_eq(RSC_STOP, action->task)) {
	crm_debug_2("Stopping due to: %s", action->uuid);
	*stopping = TRUE;

	} else if(safe_str_eq(RSC_START, action->task)) {
	if(action->runnable == FALSE) {
	crm_debug_3("Skipping pseudo-op: %s run=%d, pseudo=%d",
	action->uuid, action->runnable, action->pseudo);
	} else {
	crm_debug_2("Starting due to: %s", action->uuid);
	crm_debug_3("%s run=%d, pseudo=%d",
	action->uuid, action->runnable, action->pseudo);
	*starting = TRUE;
	}
	}
	);

	}

	static action_t *
	find_rsc_action(resource_t rsc, const char key, gboolean active_only, GListPtr *list)
	{
	action_t *match = NULL;
	GListPtr possible = NULL;
	GListPtr active = NULL;
	possible = find_actions(rsc->actions, key, NULL);

	if(active_only) {
	slist_iter(op, action_t, possible, lpc,
	if(op->optional == FALSE) {
	active = g_list_append(active, op);
	}
	);

	if(active && g_list_length(active) == 1) {
	match = g_list_nth_data(active, 0);
	}

	if(list) {
	*list = active; active = NULL;
	}

	} else if(possible && g_list_length(possible) == 1) {
	match = g_list_nth_data(possible, 0);

	} if(list) {
	*list = possible; possible = NULL;
	}

	if(possible) {
	g_list_free(possible);
	}
	if(active) {
	g_list_free(active);
	}

	return match;
	}

	static void
	child_ordering_constraints(resource_t rsc, pe_working_set_t data_set)
	{
	char *key = NULL;
	action_t *stop = NULL;
	action_t *start = NULL;
	action_t *last_stop = NULL;
	action_t *last_start = NULL;
	gboolean active_only = TRUE; /* change to false to get the old behavior */
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	if(clone_data->ordered == FALSE) {
	return;
	}

	slist_iter(
	child, resource_t, rsc->children, lpc,

	key = stop_key(child);
	stop = find_rsc_action(child, key, active_only, NULL);
	crm_free(key);

	key = start_key(child);
	start = find_rsc_action(child, key, active_only, NULL);
	crm_free(key);

	if(stop) {
	if(last_stop) {
	/* child/child relative stop */
	order_actions(stop, last_stop, pe_order_implies_left);
	}
	last_stop = stop;
	}

	if(start) {
	if(last_start) {
	/* child/child relative start */
	order_actions(last_start, start, pe_order_implies_left);
	}
	last_start = start;
	}
	);
	}

	void clone_create_actions(resource_t rsc, pe_working_set_t data_set)
	{
	gboolean child_active = FALSE;
	gboolean child_starting = FALSE;
	gboolean child_stopping = FALSE;

	action_t *stop = NULL;
	action_t *stopped = NULL;

	action_t *start = NULL;
	action_t *started = NULL;

	resource_t *last_start_rsc = NULL;
	resource_t *last_stop_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	crm_debug_2("Creating actions for %s", rsc->id);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->create_actions(child_rsc, data_set);
	clone_update_pseudo_status(
	child_rsc, &child_stopping, &child_starting, &child_active);

	if(is_set(child_rsc->flags, pe_rsc_starting)) {
	last_start_rsc = child_rsc;
	}
	if(is_set(child_rsc->flags, pe_rsc_stopping)) {
	last_stop_rsc = child_rsc;
	}
	);

	/* start */
	start = start_action(rsc, NULL, !child_starting);
	started = custom_action(rsc, started_key(rsc),
	RSC_STARTED, NULL, !child_starting, TRUE, data_set);

	start->pseudo = TRUE;
	start->runnable = TRUE;
	started->pseudo = TRUE;
	started->priority = INFINITY;

	if(child_active \|\| child_starting) {
	started->runnable = TRUE;
	}

	child_ordering_constraints(rsc, data_set);
	child_starting_constraints(clone_data, rsc, NULL, last_start_rsc, data_set);
	clone_data->start_notify = create_notification_boundaries(rsc, RSC_START, start, started, data_set);

	/* stop */
	stop = stop_action(rsc, NULL, !child_stopping);
	stopped = custom_action(rsc, stopped_key(rsc),
	RSC_STOPPED, NULL, !child_stopping, TRUE, data_set);

	stop->pseudo = TRUE;
	stop->runnable = TRUE;
	stopped->pseudo = TRUE;
	stopped->runnable = TRUE;
	stopped->priority = INFINITY;
	child_stopping_constraints(clone_data, rsc, NULL, last_stop_rsc, data_set);
	clone_data->stop_notify = create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);

	if(clone_data->stop_notify && clone_data->start_notify) {
	order_actions(clone_data->stop_notify->post_done, clone_data->start_notify->pre, pe_order_optional);
	}
	}

	void
	child_starting_constraints(
	clone_variant_data_t *clone_data,
	resource_t rsc, resource_t child, resource_t *last,
	pe_working_set_t *data_set)
	{
	if(child == NULL && last == NULL) {
	crm_debug("%s has no active children", rsc->id);
	return;
	}

	if(child != NULL) {
	order_start_start(
	rsc, child, pe_order_runnable_left\|pe_order_implies_left_printed);

	new_rsc_order(child, RSC_START, rsc, RSC_STARTED,
	pe_order_implies_right_printed, data_set);
	}

	if(FALSE && clone_data->ordered) {
	if(child == NULL) {
	/* last child start before global started */
	new_rsc_order(last, RSC_START, rsc, RSC_STARTED,
	pe_order_runnable_left, data_set);

	} else if(last == NULL) {
	/* global start before first child start */
	order_start_start(
	rsc, child, pe_order_implies_left);

	} else {
	/* child/child relative start */
	order_start_start(last, child, pe_order_implies_left);
	}
	}
	}

	void
	child_stopping_constraints(
	clone_variant_data_t *clone_data,
	resource_t rsc, resource_t child, resource_t *last,
	pe_working_set_t *data_set)
	{
	if(child == NULL && last == NULL) {
	crm_debug("%s has no active children", rsc->id);
	return;
	}

	if(child != NULL) {
	order_stop_stop(rsc, child, pe_order_shutdown\|pe_order_implies_left_printed);

	new_rsc_order(child, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_implies_right_printed, data_set);
	}

	if(FALSE && clone_data->ordered) {
	if(last == NULL) {
	/* first child stop before global stopped */
	new_rsc_order(child, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_runnable_left, data_set);

	} else if(child == NULL) {
	/* global stop before last child stop */
	order_stop_stop(
	rsc, last, pe_order_implies_left);
	} else {
	/* child/child relative stop */
	order_stop_stop(child, last, pe_order_implies_left);
	}
	}
	}


	void
	clone_internal_constraints(resource_t rsc, pe_working_set_t data_set)
	{
	resource_t *last_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	native_internal_constraints(rsc, data_set);

	/* global stop before stopped */
	new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set);

	/* global start before started */
	new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set);

	/* global stopped before start */
	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	child_starting_constraints(
	clone_data, rsc, child_rsc, last_rsc, data_set);

	child_stopping_constraints(
	clone_data, rsc, child_rsc, last_rsc, data_set);

	last_rsc = child_rsc;
	);
	}

	resource_t*
	find_compatible_child(
	resource_t local_child, resource_t rsc, enum rsc_role_e filter, gboolean current)
	{
	node_t *local_node = NULL;
	node_t *node = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	local_node = local_child->fns->location(local_child, NULL, current);
	if(local_node == NULL) {
	crm_debug("Can't colocate unrunnable child %s with %s",
	local_child->id, rsc->id);
	return NULL;
	}

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, current);
	node = child_rsc->fns->location(child_rsc, NULL, current);

	if(filter != RSC_ROLE_UNKNOWN && next_role != filter) {
	crm_debug_2("Filtered %s", child_rsc->id);
	continue;
	}

	if(node && local_node && node->details == local_node->details) {
	crm_info("Colocating %s with %s on %s",
	local_child->id, child_rsc->id, node->details->uname);
	return child_rsc;
	}
	);
	crm_debug("Can't colocate child %s with %s",
	local_child->id, rsc->id);
	return NULL;
	}

	void clone_rsc_colocation_lh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	gboolean do_interleave = FALSE;
	resource_t *rsc = constraint->rsc_lh;
	clone_variant_data_t *clone_data = NULL;
	clone_variant_data_t *clone_data_rh = NULL;

	if(rsc == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if(constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;

	} else {
	crm_debug_4("Processing constraints from %s", rsc->id);
	}

	get_clone_variant_data(clone_data, rsc);

	if(constraint->rsc_rh->variant == pe_clone
	\|\| constraint->rsc_rh->variant == pe_master) {
	get_clone_variant_data(
	clone_data_rh, constraint->rsc_rh);
	if(clone_data->clone_node_max
	!= clone_data_rh->clone_node_max) {
	crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION
	" %s and %s because"
	" they do not support the same number of"
	" resources per node",
	constraint->rsc_lh->id, constraint->rsc_rh->id);

	/* only the LHS side needs to be labeled as interleave */
	} else if(clone_data->interleave) {
	do_interleave = TRUE;

	} else if(constraint->score >= INFINITY) {
	GListPtr lhs = NULL, rhs = NULL;
	lhs = rsc_lh->allowed_nodes;

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,
	node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if(chosen != NULL) {
	rhs = g_list_append(rhs, chosen);
	}
	);

	rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs);

	pe_free_shallow_adv(rhs, FALSE);
	pe_free_shallow(lhs);
	return;
	}

	} else if(constraint->score >= INFINITY) {
	crm_config_err("Manditory co-location of clones (%s) with other"
	" non-clone (%s) resources is not supported",
	rsc_lh->id, rsc_rh->id);
	return;
	}

	if(do_interleave) {
	resource_t *rh_child = NULL;

	slist_iter(lh_child, resource_t, rsc->children, lpc,

	CRM_ASSERT(lh_child != NULL);
	rh_child = find_compatible_child(
	lh_child, rsc_rh, RSC_ROLE_UNKNOWN, FALSE);
	if(rh_child == NULL) {
	crm_debug_2("No match found for %s", lh_child->id);
	continue;
	}
	crm_debug("Interleaving %s with %s", lh_child->id, rh_child->id);
	lh_child->cmds->rsc_colocation_lh(
	lh_child, rh_child, constraint);
	);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->rsc_colocation_lh(child_rsc, constraint->rsc_rh, constraint);
	);
	}

	void clone_rsc_colocation_rh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	clone_variant_data_t *clone_data = NULL;
	CRM_CHECK(rsc_lh != NULL, return);
	CRM_CHECK(rsc_lh->variant == pe_native, return);

	get_clone_variant_data(clone_data, rsc_rh);

	crm_debug_3("Processing constraint %s: %d", constraint->id, constraint->score);

	if(rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;

	} else if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
	crm_debug_3("%s is still provisional", rsc_rh->id);
	return;

	} else if(constraint->score >= INFINITY) {
	GListPtr lhs = NULL, rhs = NULL;
	lhs = rsc_lh->allowed_nodes;

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,
	node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if(chosen != NULL) {
	rhs = g_list_append(rhs, chosen);
	}
	);

	rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs);

	pe_free_shallow_adv(rhs, FALSE);
	pe_free_shallow(lhs);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,

	child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
	);
	}

	/*

	Clone <-> Clone ordering

	S : Start(ed)
	S' : Stop(ped)
	P : Promote(d)
	D : Demote(d)

	Started == Demoted

	First A then B
	A:0 B:0
	Old New Old New

	S' S' S S'
	S' S' S' -
	S' S S S+
	S' S S' S
	S S' S S'
	S S' S' -
	S S S -
	S S S' S

	S' S' P S'
	S' S' S' -
	S' P P P+
	S' P S' P
	P S' P S'
	P S' S' -
	P P P -
	P P S' P

	D D P D
	D D D -
	D P P P+
	D P D P
	P D P D
	P D D -
	P P P -
	P P D P

	Clone <-> Primitive ordering

	S : Start(ed)
	S' : Stop(ped)
	P : Promote(d)
	D : Demote(d)
	F : False
	T : True
	F' : A good idea?

	Started == Demoted

	First A then B
	A:0 B
	Old New Old Create Constraint

	S' S' S F
	S' S' S' F'
	S S' S T
	S S' S' F
	S' S S T
	S' S S' T
	S S S F'
	S S S' T

	S' S' S F
	S' S' S' F'
	P S' S T
	P S' S' F
	S' P S T
	S' P S' T
	P P S F'
	P P S' F

	S' S' S F
	S' S' S' F'
	D S' S T
	D S' S' F
	S' D S T
	S' D S' T
	D D S F'
	D D S' T

	*/
	static gboolean detect_restart(resource_t *rsc)
	{
	gboolean restart = FALSE;

	/* Look for restarts */
	action_t *start = NULL;
	char *key = start_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, NULL);
	crm_free(key);

	if(possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	if(start != NULL && start->optional == FALSE) {
	restart = TRUE;
	crm_debug_2("Detected a restart for %s", rsc->id);
	}

	/* Otherwise, look for moves */
	if(restart == FALSE) {
	GListPtr old_hosts = NULL;
	GListPtr new_hosts = NULL;
	GListPtr intersection = NULL;

	rsc->fns->location(rsc, &old_hosts, TRUE);
	rsc->fns->location(rsc, &new_hosts, FALSE);
	intersection = node_list_and(old_hosts, new_hosts, FALSE);

	if(intersection == NULL) {
	restart = TRUE; /* Actually a move but the result is the same */
	crm_debug_2("Detected a move for %s", rsc->id);
	}

	g_list_free(intersection);
	g_list_free(old_hosts);
	g_list_free(new_hosts);
	}

	return restart;
	}

	static void clone_rsc_order_lh_non_clone(resource_t rsc, order_constraint_t order, pe_working_set_t *data_set)
	{
	GListPtr hosts = NULL;
	GListPtr rh_hosts = NULL;
	GListPtr intersection = NULL;

	const char *reason = "unknown";
	enum action_tasks task = start_rsc;
	enum rsc_role_e lh_role = RSC_ROLE_STARTED;

	int any_ordered = 0;
	gboolean down_stack = TRUE;

	crm_debug_2("Clone-to-* ordering: %s -> %s 0x%.6x",
	order->lh_action_task, order->rh_action_task, order->type);

	if(strstr(order->rh_action_task, "_"RSC_STOP"_0")
	\|\| strstr(order->rh_action_task, "_"RSC_STOPPED"_0")) {
	task = stop_rsc;
	reason = "down activity";
	lh_role = RSC_ROLE_STOPPED;
	order->rh_rsc->fns->location(order->rh_rsc, &rh_hosts, down_stack);

	} else if(strstr(order->rh_action_task, "_"RSC_DEMOTE"_0")
	\|\| strstr(order->rh_action_task, "_"RSC_DEMOTED"_0")) {
	task = action_demote;
	reason = "demotion activity";
	lh_role = RSC_ROLE_SLAVE;
	order->rh_rsc->fns->location(order->rh_rsc, &rh_hosts, down_stack);

	} else if(strstr(order->lh_action_task, "_"RSC_PROMOTE"_0")
	\|\| strstr(order->lh_action_task, "_"RSC_PROMOTED"_0")) {
	task = action_promote;
	down_stack = FALSE;
	reason = "promote activity";
	order->rh_rsc->fns->location(order->rh_rsc, &rh_hosts, down_stack);
	lh_role = RSC_ROLE_MASTER;

	} else if(strstr(order->rh_action_task, "_"RSC_START"_0")
	\|\| strstr(order->rh_action_task, "_"RSC_STARTED"_0")) {
	task = start_rsc;
	down_stack = FALSE;
	reason = "up activity";
	order->rh_rsc->fns->location(order->rh_rsc, &rh_hosts, down_stack);
	/* if(order->rh_rsc->variant > pe_clone) { */
	/* lh_role = RSC_ROLE_SLAVE; */
	/* } */

	} else {
	crm_err("Unknown task: %s", order->rh_action_task);
	return;
	}

	/* slist_iter(h, node_t, rh_hosts, llpc, crm_info("RHH: %s", h->details->uname)); */

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	gboolean create = FALSE;
	gboolean restart = FALSE;
	enum rsc_role_e lh_role_new = child_rsc->fns->state(child_rsc, FALSE);
	enum rsc_role_e lh_role_old = child_rsc->fns->state(child_rsc, TRUE);
	enum rsc_role_e child_role = child_rsc->fns->state(child_rsc, down_stack);

	crm_debug_4("Testing %s->%s for %s: %s vs. %s %s",
	order->lh_action_task, order->rh_action_task, child_rsc->id,
	role2text(lh_role), role2text(child_role), order->lh_action_task);

	if(rh_hosts == NULL) {
	crm_debug_3("Terminating search: %s.%d list is empty: no possible %s",
	order->rh_rsc->id, down_stack, reason);
	break;
	}

	if(lh_role_new == lh_role_old) {
	restart = detect_restart(child_rsc);
	if(restart == FALSE) {
	crm_debug_3("Ignoring %s->%s for %s: no relevant %s (no role change)",
	order->lh_action_task, order->rh_action_task, child_rsc->id, reason);
	continue;
	}
	}

	hosts = NULL;
	child_rsc->fns->location(child_rsc, &hosts, down_stack);
	intersection = node_list_and(hosts, rh_hosts, FALSE);
	/* slist_iter(h, node_t, hosts, llpc, crm_info("H: %s %s", child_rsc->id, h->details->uname)); */
	if(intersection == NULL) {
	crm_debug_3("Ignoring %s->%s for %s: no relevant %s",
	order->lh_action_task, order->rh_action_task, child_rsc->id, reason);
	g_list_free(hosts);
	continue;
	}

	if(restart) {
	reason = "restart";
	create = TRUE;

	} else if(down_stack) {
	if(lh_role_old > lh_role) {
	create = TRUE;
	}

	} else if(down_stack == FALSE) {
	if(lh_role_old < lh_role) {
	create = TRUE;
	}

	} else {
	any_ordered++;
	reason = "role";
	crm_debug_4("Role: %s->%s for %s: %s vs. %s %s",
	order->lh_action_task, order->rh_action_task, child_rsc->id,
	role2text(lh_role_old), role2text(lh_role), order->lh_action_task);

	}

	if(create) {
	char *task = order->lh_action_task;

	any_ordered++;
	crm_debug("Enforcing %s->%s for %s on %s: found %s",
	order->lh_action_task, order->rh_action_task, child_rsc->id,
	((node_t*)intersection->data)->details->uname, reason);

	order->lh_action_task = convert_non_atomic_task(task, child_rsc, TRUE, FALSE);
	child_rsc->cmds->rsc_order_lh(child_rsc, order, data_set);
	crm_free(order->lh_action_task);
	order->lh_action_task = task;
	}

	crm_debug_3("Processed %s->%s for %s on %s: %s",
	order->lh_action_task, order->rh_action_task, child_rsc->id,
	((node_t*)intersection->data)->details->uname, reason);

	/* slist_iter(h, node_t, hosts, llpc, */
	/* crm_info("H: %s %s", child_rsc->id, h->details->uname)); */

	g_list_free(intersection);
	g_list_free(hosts);

	);

	g_list_free(rh_hosts);
	if(any_ordered == 0 && down_stack == FALSE) {
	order->lh_action_task = convert_non_atomic_task(order->lh_action_task, rsc, TRUE, TRUE);
	native_rsc_order_lh(rsc, order, data_set);
	}
	order->type = pe_order_optional;
	}

	void clone_rsc_order_lh(resource_t rsc, order_constraint_t order, pe_working_set_t *data_set)
	{
	resource_t *r1 = NULL;
	resource_t *r2 = NULL;
	gboolean do_interleave = FALSE;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_4("%s->%s", order->lh_action_task, order->rh_action_task);
	if(order->rh_rsc == NULL) {
	order->lh_action_task = convert_non_atomic_task(order->lh_action_task, rsc, FALSE, TRUE);
	native_rsc_order_lh(rsc, order, data_set);
	return;
	}

	r1 = uber_parent(rsc);
	r2 = uber_parent(order->rh_rsc);

	if(r1 == r2) {
	native_rsc_order_lh(rsc, order, data_set);
	return;
	}

	if(order->rh_rsc->variant > pe_group) {
	clone_variant_data_t *clone_data_rh = NULL;
	get_clone_variant_data(clone_data_rh, order->rh_rsc);
	if(clone_data->clone_node_max != clone_data_rh->clone_node_max) {
	crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION
	" %s and %s because they do not support the same"
	" number of resources per node",
	rsc->id, order->rh_rsc->id);

	/* only the LHS side needs to be labeled as interleave */
	} else if(clone_data->interleave) {
	do_interleave = TRUE;
	}
	}


	if(order->rh_rsc == NULL) {
	do_interleave = FALSE;
	}

	if(do_interleave) {
	resource_t *lh_child = NULL;
	resource_t *rh_saved = order->rh_rsc;
	gboolean current = FALSE;

	if(strstr(order->lh_action_task, "_stop_0") \|\| strstr(order->lh_action_task, "_demote_0")) {
	current = TRUE;
	}

	slist_iter(
	rh_child, resource_t, rh_saved->children, lpc,

	CRM_ASSERT(rh_child != NULL);
	lh_child = find_compatible_child(rh_child, rsc, RSC_ROLE_UNKNOWN, current);
	if(lh_child == NULL) {
	crm_debug_2("No match found for %s", rh_child->id);
	continue;
	}
	crm_notice("Interleaving %s with %s", lh_child->id, rh_child->id);
	order->rh_rsc = rh_child;
	lh_child->cmds->rsc_order_lh(lh_child, order, data_set);
	order->rh_rsc = rh_saved;
	);

	} else {

	#if 0
	if(order->type != pe_order_optional) {
	crm_debug("Upgraded ordering constraint %d - 0x%.6x", order->id, order->type);
	native_rsc_order_lh(rsc, order, data_set);
	}
	#endif

	if(order->rh_rsc->variant < pe_clone) {
	clone_rsc_order_lh_non_clone(rsc, order, data_set);

	} else if(order->type & pe_order_implies_left) {
	if(rsc->variant == order->rh_rsc->variant) {
	crm_debug_2("Clone-to-clone ordering: %s -> %s 0x%.6x",
	order->lh_action_task, order->rh_action_task, order->type);
	/* stop instances on the same nodes as stopping RHS instances */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	native_rsc_order_lh(child_rsc, order, data_set);
	);
	} else {
	/* stop everything */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	native_rsc_order_lh(child_rsc, order, data_set);
	);
	}
	}
	}

	if(do_interleave == FALSE \|\| clone_data->ordered) {
	order->lh_action_task = convert_non_atomic_task(order->lh_action_task, rsc, FALSE, TRUE);
	native_rsc_order_lh(rsc, order, data_set);
	}

	if(is_set(rsc->flags, pe_rsc_notify)) {
	order->type = pe_order_optional;
	order->lh_action_task = convert_non_atomic_task(order->lh_action_task, rsc, TRUE, TRUE);
	native_rsc_order_lh(rsc, order, data_set);
	}
	}

	static void clone_rsc_order_rh_non_clone(
	resource_t lh_p, action_t lh_action, resource_t rsc, order_constraint_t order)
	{
	GListPtr hosts = NULL;
	GListPtr lh_hosts = NULL;
	GListPtr intersection = NULL;
	const char *reason = "unknown";

	gboolean restart = FALSE;
	gboolean down_stack = TRUE;

	enum rsc_role_e rh_role = RSC_ROLE_STARTED;
	enum action_tasks task = start_rsc;

	enum rsc_role_e lh_role_new = lh_p->fns->state(lh_p, FALSE);
	enum rsc_role_e lh_role_old = lh_p->fns->state(lh_p, TRUE);
	-
	+
	+ /* Make sure the pre-req will be active */
	+ if(order->type & pe_order_runnable_left) {
	+ lh_p->fns->location(lh_p, &lh_hosts, FALSE);
	+ if(lh_hosts == NULL) {
	+ crm_debug("Terminating search: Pre-requisite %s of %s is unrunnable", lh_p->id, rsc->id);
	+ native_rsc_order_rh(lh_action, rsc, order);
	+ return;
	+ }
	+ }
	+
	if(strstr(order->lh_action_task, "_"RSC_STOP"_0")
	\|\| strstr(order->lh_action_task, "_"RSC_STOPPED"_0")) {
	task = stop_rsc;
	reason = "down activity";
	rh_role = RSC_ROLE_STOPPED;
	lh_p->fns->location(lh_p, &lh_hosts, down_stack);

	/* These actions are not possible for non-clones
	} else if(strstr(order->lh_action_task, "_"RSC_DEMOTE"_0")
	\|\| strstr(order->lh_action_task, "_"RSC_DEMOTED"_0")) {
	task = action_demote;
	rh_role = RSC_ROLE_SLAVE;
	reason = "demotion activity";
	lh_p->fns->location(lh_p, &lh_hosts, down_stack);

	} else if(strstr(order->lh_action_task, "_"RSC_PROMOTE"_0")
	\|\| strstr(order->lh_action_task, "_"RSC_PROMOTED"_0")) {
	task = action_promote;
	down_stack = FALSE;
	reason = "promote activity";
	lh_p->fns->location(lh_p, &lh_hosts, down_stack);
	rh_role = RSC_ROLE_MASTER;
	*/
	} else if(strstr(order->lh_action_task, "_"RSC_START"_0")
	\|\| strstr(order->lh_action_task, "_"RSC_STARTED"_0")) {
	task = start_rsc;
	down_stack = FALSE;
	reason = "up activity";
	lh_p->fns->location(lh_p, &lh_hosts, down_stack);

	} else {
	crm_err("Unknown action: %s", order->lh_action_task);
	return;
	}

	if(lh_role_new == lh_role_old) {
	restart = detect_restart(lh_action->rsc);

	if(FALSE && restart == FALSE) {
	crm_debug_3("Ignoring %s->%s for %s: no relevant %s (no role change)",
	lh_action->task, order->lh_action_task, lh_p->id, reason);
	goto cleanup;
	}
	}

	/* slist_iter(h, node_t, lh_hosts, llpc, crm_info("LHH: %s", h->details->uname)); */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	gboolean create = FALSE;
	enum rsc_role_e child_role = child_rsc->fns->state(child_rsc, down_stack);

	crm_debug_4("Testing %s->%s for %s: %s vs. %s %s",
	lh_action->task, order->lh_action_task, child_rsc->id,
	role2text(rh_role), role2text(child_role), order->lh_action_task);

	if(lh_hosts == NULL) {
	crm_debug_3("Terminating search: %s.%d list is empty: no possible %s",
	order->rh_rsc->id, down_stack, reason);
	break;
	}

	hosts = NULL;
	child_rsc->fns->location(child_rsc, &hosts, down_stack);
	intersection = node_list_and(hosts, lh_hosts, FALSE);
	if(intersection == NULL) {
	crm_debug_3("Ignoring %s->%s for %s: no relevant %s",
	lh_action->task, order->lh_action_task, child_rsc->id, reason);
	g_list_free(hosts);
	continue;
	}

	/* slist_iter(h, node_t, hosts, llpc, crm_info("H: %s %s", child_rsc->id, h->details->uname)); */
	if(restart) {
	reason = "restart";
	create = TRUE;

	} else if(down_stack && lh_role_old >= rh_role) {
	create = TRUE;

	} else if(down_stack == FALSE && lh_role_old <= rh_role) {
	create = TRUE;

	} else {
	reason = "role";
	}

	if(create) {
	enum pe_ordering type = order->type;
	child_rsc->cmds->rsc_order_rh(lh_action, child_rsc, order);
	order->type = pe_order_optional;
	native_rsc_order_rh(lh_action, rsc, order);
	order->type = type;
	}

	crm_debug_3("Processed %s->%s for %s on %s: found %s%s",
	lh_action->task, order->lh_action_task, child_rsc->id,
	((node_t*)intersection->data)->details->uname, reason, create?" - enforced":"");

	/* slist_iter(h, node_t, hosts, llpc, */
	/* crm_info("H: %s %s", child_rsc->id, h->details->uname)); */

	g_list_free(intersection);
	g_list_free(hosts);
	);
	cleanup:
	g_list_free(lh_hosts);
	}

	void clone_rsc_order_rh(
	action_t lh_action, resource_t rsc, order_constraint_t *order)
	{
	enum pe_ordering type = order->type;
	clone_variant_data_t *clone_data = NULL;
	resource_t *lh_p = uber_parent(lh_action->rsc);

	get_clone_variant_data(clone_data, rsc);
	crm_debug_2("%s->%s", order->lh_action_task, order->rh_action_task);

	if(safe_str_eq(CRM_OP_PROBED, lh_action->uuid)) {
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->rsc_order_rh(lh_action, child_rsc, order);
	);

	if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED
	&& rsc->fns->state(rsc, FALSE) > RSC_ROLE_STOPPED) {
	order->type \|= pe_order_implies_right;
	}

	} else if(lh_p && lh_p != rsc && lh_p->variant < pe_clone) {
	clone_rsc_order_rh_non_clone(lh_p, lh_action, rsc, order);
	return;
	}

	native_rsc_order_rh(lh_action, rsc, order);
	order->type = type;
	}

	void clone_rsc_location(resource_t rsc, rsc_to_node_t constraint)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_3("Processing location constraint %s for %s",
	constraint->id, rsc->id);

	native_rsc_location(rsc, constraint);
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->rsc_location(child_rsc, constraint);
	);
	}


	void clone_expand(resource_t rsc, pe_working_set_t data_set)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_2("Processing actions from %s", rsc->id);

	if(clone_data->start_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->start_notify);
	expand_notification_data(clone_data->start_notify);
	create_notifications(rsc, clone_data->start_notify, data_set);
	}

	if(clone_data->stop_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->stop_notify);
	expand_notification_data(clone_data->stop_notify);
	create_notifications(rsc, clone_data->stop_notify, data_set);
	}

	if(clone_data->promote_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->promote_notify);
	expand_notification_data(clone_data->promote_notify);
	create_notifications(rsc, clone_data->promote_notify, data_set);
	}

	if(clone_data->demote_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->demote_notify);
	expand_notification_data(clone_data->demote_notify);
	create_notifications(rsc, clone_data->demote_notify, data_set);
	}

	/* Now that the notifcations have been created we can expand the children */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->expand(child_rsc, data_set));

	native_expand(rsc, data_set);

	/* The notifications are in the graph now, we can destroy the notify_data */
	free_notification_data(clone_data->demote_notify);
	free_notification_data(clone_data->stop_notify);
	free_notification_data(clone_data->start_notify);
	free_notification_data(clone_data->promote_notify);
	}


	static gint sort_rsc_id(gconstpointer a, gconstpointer b)
	{
	const resource_t resource1 = (const resource_t)a;
	const resource_t resource2 = (const resource_t)b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	return strcmp(resource1->id, resource2->id);
	}

	static resource_t find_instance_on(resource_t rsc, node_t *node)
	{
	slist_iter(child, resource_t, rsc->children, lpc,
	GListPtr known_list = NULL;
	rsc_known_on(child, &known_list);
	slist_iter(known, node_t, known_list, lpc2,
	if(node->details == known->details) {
	g_list_free(known_list);
	return child;
	}
	);
	g_list_free(known_list);
	);
	return NULL;
	}

	gboolean
	clone_create_probe(resource_t rsc, node_t node, action_t *complete,
	gboolean force, pe_working_set_t *data_set)
	{
	gboolean any_created = FALSE;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	rsc->children = g_list_sort(rsc->children, sort_rsc_id);
	if(rsc->children == NULL) {
	pe_warn("Clone %s has no children", rsc->id);
	return FALSE;
	}

	if(is_not_set(rsc->flags, pe_rsc_unique)
	&& clone_data->clone_node_max == 1) {
	/* only look for one copy */
	resource_t *child = NULL;

	/* Try whoever we probed last time */
	child = find_instance_on(rsc, node);
	if(child) {
	return child->cmds->create_probe(
	child, node, complete, force, data_set);
	}

	/* Try whoever we plan on starting there */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	node_t *local_node = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if(local_node == NULL) {
	continue;
	}

	if(local_node->details == node->details) {
	return child_rsc->cmds->create_probe(
	child_rsc, node, complete, force, data_set);
	}
	);

	/* Fall back to the first clone instance */
	child = rsc->children->data;
	return child->cmds->create_probe(child, node, complete, force, data_set);
	}
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	if(child_rsc->cmds->create_probe(
	child_rsc, node, complete, force, data_set)) {
	any_created = TRUE;
	}

	if(any_created
	&& is_not_set(rsc->flags, pe_rsc_unique)
	&& clone_data->clone_node_max == 1) {
	/* only look for one copy (clone :0) */
	break;
	}
	);

	return any_created;
	}
	diff --git a/pengine/regression.sh b/pengine/regression.sh
	index 89cbec81a3..de65017552 100755
	--- a/pengine/regression.sh
	+++ b/pengine/regression.sh
	@@ -1,341 +1,342 @@
	#!/bin/bash

	# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public
	# License as published by the Free Software Foundation; either
	# version 2.1 of the License, or (at your option) any later version.
	#
	# This software is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public
	# License along with this library; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	#
	if [ -x /usr/bin/valgrind ]; then
	export G_SLICE=always-malloc
	# VALGRIND_CMD="valgrind -q --show-reachable=no --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=./ptest.supp"
	fi

	. regression.core.sh
	create_mode="true"
	echo Generating test outputs for these tests...
	# do_test file description
	echo Done.
	echo ""

	echo Performing the following tests...
	create_mode="false"

	echo ""
	do_test simple1 "Offline "
	do_test simple2 "Start "
	do_test simple3 "Start 2 "
	do_test simple4 "Start Failed"
	do_test simple6 "Stop Start "
	do_test simple7 "Shutdown "
	#do_test simple8 "Stonith "
	#do_test simple9 "Lower version"
	#do_test simple10 "Higher version"
	do_test simple11 "Priority (ne)"
	do_test simple12 "Priority (eq)"
	do_test simple8 "Stickiness"

	echo ""
	do_test params-0 "Params: No change"
	do_test params-1 "Params: Changed"
	do_test params-2 "Params: Resource definition"
	do_test params-4 "Params: Reload"
	do_test novell-251689 "Resource definition change + target_role=stopped"
	do_test bug-lf-2106 "Restart all anonymous clone instances after config change"

	echo ""
	do_test orphan-0 "Orphan ignore"
	do_test orphan-1 "Orphan stop"

	echo ""
	do_test target-0 "Target Role : baseline"
	do_test target-1 "Target Role : master"
	do_test target-2 "Target Role : invalid"

	echo ""
	do_test date-1 "Dates" -d "2005-020"
	do_test date-2 "Date Spec - Pass" -d "2005-020T12:30"
	do_test date-3 "Date Spec - Fail" -d "2005-020T11:30"
	do_test probe-0 "Probe (anon clone)"
	do_test probe-1 "Pending Probe"
	do_test probe-2 "Correctly re-probe cloned groups"
	do_test probe-3 "Probe (pending node)"
	do_test probe-4 "Probe (pending node + stopped resource)" --rc 5
	do_test standby "Standby"
	do_test comments "Comments"

	echo ""
	do_test rsc_dep1 "Must not "
	do_test rsc_dep3 "Must "
	do_test rsc_dep5 "Must not 3 "
	do_test rsc_dep7 "Must 3 "
	do_test rsc_dep10 "Must (but cant)"
	do_test rsc_dep2 "Must (running) "
	do_test rsc_dep8 "Must (running : alt) "
	do_test rsc_dep4 "Must (running + move)"
	do_test asymmetric "Asymmetric - require explicit location constraints"

	echo ""
	do_test order1 "Order start 1 "
	do_test order2 "Order start 2 "
	do_test order3 "Order stop "
	do_test order4 "Order (multiple) "
	do_test order5 "Order (move) "
	do_test order6 "Order (move w/ restart) "
	do_test order7 "Order (manditory) "
	do_test order-optional "Order (score=0) "
	do_test order-required "Order (score=INFINITY) "
	do_test bug-lf-2171 "Prevent group start when clone is stopped"
	do_test order-clone "Clone ordering should be able to prevent startup of dependant clones"
	do_test order-sets "Ordering for resource sets"

	echo ""
	do_test coloc-loop "Colocation - loop"
	do_test coloc-many-one "Colocation - many-to-one"
	do_test coloc-list "Colocation - many-to-one with list"
	do_test coloc-group "Colocation - groups"
	do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
	do_test coloc-attr "Colocation based on node attributes"
	do_test coloc-negative-group "Negative colocation with a group"

	#echo ""
	#do_test agent1 "version: lt (empty)"
	#do_test agent2 "version: eq "
	#do_test agent3 "version: gt "

	echo ""
	do_test attrs1 "string: eq (and) "
	do_test attrs2 "string: lt / gt (and)"
	do_test attrs3 "string: ne (or) "
	do_test attrs4 "string: exists "
	do_test attrs5 "string: not_exists "
	do_test attrs6 "is_dc: true "
	do_test attrs7 "is_dc: false "
	do_test attrs8 "score_attribute "

	echo ""
	do_test mon-rsc-1 "Schedule Monitor - start"
	do_test mon-rsc-2 "Schedule Monitor - move "
	do_test mon-rsc-3 "Schedule Monitor - pending start "
	do_test mon-rsc-4 "Schedule Monitor - move/pending start"

	echo ""
	do_test rec-rsc-0 "Resource Recover - no start "
	do_test rec-rsc-1 "Resource Recover - start "
	do_test rec-rsc-2 "Resource Recover - monitor "
	do_test rec-rsc-3 "Resource Recover - stop - ignore"
	do_test rec-rsc-4 "Resource Recover - stop - block "
	do_test rec-rsc-5 "Resource Recover - stop - fence "
	do_test rec-rsc-6 "Resource Recover - multiple - restart"
	do_test rec-rsc-7 "Resource Recover - multiple - stop "
	do_test rec-rsc-8 "Resource Recover - multiple - block "
	do_test rec-rsc-9 "Resource Recover - group/group"

	echo ""
	do_test quorum-1 "No quorum - ignore"
	do_test quorum-2 "No quorum - freeze"
	do_test quorum-3 "No quorum - stop "
	do_test quorum-4 "No quorum - start anyway"
	do_test quorum-5 "No quorum - start anyway (group)"
	do_test quorum-6 "No quorum - start anyway (clone)"

	echo ""
	do_test rec-node-1 "Node Recover - Startup - no fence"
	do_test rec-node-2 "Node Recover - Startup - fence "
	do_test rec-node-3 "Node Recover - HA down - no fence"
	do_test rec-node-4 "Node Recover - HA down - fence "
	do_test rec-node-5 "Node Recover - CRM down - no fence"
	do_test rec-node-6 "Node Recover - CRM down - fence "
	do_test rec-node-7 "Node Recover - no quorum - ignore "
	do_test rec-node-8 "Node Recover - no quorum - freeze "
	do_test rec-node-9 "Node Recover - no quorum - stop "
	do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
	do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
	do_test rec-node-12 "Node Recover - nothing active - fence "
	do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
	do_test rec-node-15 "Node Recover - unknown lrm section"
	do_test rec-node-14 "Serialize all stonith's"

	echo ""
	do_test multi1 "Multiple Active (stop/start)"

	echo ""
	do_test migrate-stop "Migration in a stopping stack"
	do_test migrate-start "Migration in a starting stack"
	do_test migrate-stop_start "Migration in a restarting stack"
	do_test migrate-stop-complex "Migration in a complex stopping stack"
	do_test migrate-start-complex "Migration in a complex starting stack"
	do_test migrate-stop-start-complex "Migration in a complex moving stack"

	do_test migrate-1 "Migrate (migrate)"
	do_test migrate-2 "Migrate (stable)"
	do_test migrate-3 "Migrate (failed migrate_to)"
	do_test migrate-4 "Migrate (failed migrate_from)"
	do_test novell-252693 "Migration in a stopping stack"
	do_test novell-252693-2 "Migration in a starting stack"
	do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
	do_test bug-1820 "Migration in a group"
	do_test bug-1820-1 "Non-migration in a group"
	do_test migrate-5 "Primitive migration with a clone"

	#echo ""
	#do_test complex1 "Complex "

	echo ""
	do_test group1 "Group "
	do_test group2 "Group + Native "
	do_test group3 "Group + Group "
	do_test group4 "Group + Native (nothing)"
	do_test group5 "Group + Native (move) "
	do_test group6 "Group + Group (move) "
	do_test group7 "Group colocation"
	do_test group13 "Group colocation (cant run)"
	do_test group8 "Group anti-colocation"
	do_test group9 "Group recovery"
	do_test group10 "Group partial recovery"
	do_test group11 "Group target_role"
	do_test group14 "Group stop (graph terminated)"
	do_test group15 "-ve group colocation"
	do_test bug-1573 "Partial stop of a group with two children"
	do_test bug-1718 "Mandatory group ordering - Stop group_FUN"

	echo ""
	do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
	do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
	do_test inc0 "Incarnation start"
	do_test inc1 "Incarnation start order"
	do_test inc2 "Incarnation silent restart, stop, move"
	do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
	do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
	do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
	do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
	do_test inc7 "Clone colocation"
	do_test inc8 "Clone anti-colocation"
	do_test inc9 "Non-unique clone"
	do_test inc10 "Non-unique clone (stop)"
	do_test inc11 "Primitive colocation with clones"
	do_test inc12 "Clone shutdown"
	do_test cloned-group "Make sure only the correct number of cloned groups are started"
	do_test clone-no-shuffle "Dont prioritize allocation of instances that must be moved"
	do_test clone-max-zero "Orphan processing with clone-max=0"
	do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
	do_test bug-lf-2160 "Dont shuffle clones due to colocation"
	do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
	do_test bug-lf-2153 "Clone ordering constraints"
	+do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"

	echo ""
	do_test master-0 "Stopped -> Slave"
	do_test master-1 "Stopped -> Promote"
	do_test master-2 "Stopped -> Promote : notify"
	do_test master-3 "Stopped -> Promote : master location"
	do_test master-4 "Started -> Promote : master location"
	do_test master-5 "Promoted -> Promoted"
	do_test master-6 "Promoted -> Promoted (2)"
	do_test master-7 "Promoted -> Fenced"
	do_test master-8 "Promoted -> Fenced -> Moved"
	do_test master-9 "Stopped + Promotable + No quorum"
	do_test master-10 "Stopped -> Promotable : notify with monitor"
	do_test master-11 "Stopped -> Promote : colocation"
	do_test novell-239082 "Demote/Promote ordering"
	do_test novell-239087 "Stable master placement"
	do_test master-12 "Promotion based solely on rsc_location constraints"
	do_test master-13 "Include preferences of colocated resources when placing master"
	do_test master-demote "Ordering when actions depends on demoting a slave resource"
	do_test master-ordering "Prevent resources from starting that need a master"
	do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
	do_test master-group "Promotion of cloned groups"
	do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
	do_test master-failed-demote "Dont retry failed demote actions"
	do_test master-failed-demote-2 "Dont retry failed demote actions (notify=false)"
	do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
	do_test master-reattach "Re-attach to a running master"
	do_test master-allow-start "Don't include master score if it would prevent allocation"
	do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
	do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
	do_test master-role "Prevent target-role from promoting more than master-max instances"
	do_test bug-lf-2358 "Master-Master anti-colocation"

	echo ""
	do_test managed-0 "Managed (reference)"
	do_test managed-1 "Not managed - down "
	do_test managed-2 "Not managed - up "

	echo ""
	do_test interleave-0 "Interleave (reference)"
	do_test interleave-1 "coloc - not interleaved"
	do_test interleave-2 "coloc - interleaved "
	do_test interleave-3 "coloc - interleaved (2)"
	do_test interleave-pseudo-stop "Interleaved clone during stonith"
	do_test interleave-stop "Interleaved clone during stop"
	do_test interleave-restart "Interleaved clone during dependency restart"

	echo ""
	do_test notify-0 "Notify reference"
	do_test notify-1 "Notify simple"
	do_test notify-2 "Notify simple, confirm"
	do_test notify-3 "Notify move, confirm"
	do_test novell-239079 "Notification priority"
	#do_test notify-2 "Notify - 764"

	echo ""
	do_test 594 "OSDL #594"
	do_test 662 "OSDL #662"
	do_test 696 "OSDL #696"
	do_test 726 "OSDL #726"
	do_test 735 "OSDL #735"
	do_test 764 "OSDL #764"
	do_test 797 "OSDL #797"
	do_test 829 "OSDL #829"
	do_test 994 "OSDL #994"
	do_test 994-2 "OSDL #994 - with a dependant resource"
	do_test 1360 "OSDL #1360 - Clone stickiness"
	do_test 1484 "OSDL #1484 - on_fail=stop"
	do_test 1494 "OSDL #1494 - Clone stability"
	do_test unrunnable-1 "Unrunnable"
	do_test stonith-0 "Stonith loop - 1"
	do_test stonith-1 "Stonith loop - 2"
	do_test stonith-2 "Stonith loop - 3"
	do_test stonith-3 "Stonith startup"
	do_test bug-1572-1 "Recovery of groups depending on master/slave"
	do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
	do_test bug-1685 "Depends-on-master ordering"
	do_test bug-1822 "Dont promote partially active groups"
	do_test bug-pm-11 "New resource added to a m/s group"
	do_test bug-pm-12 "Recover only the failed portion of a cloned group"
	do_test bug-n-387749 "Don't shuffle clone instances"
	do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
	do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
	do_test bug-lf-1920 "Correctly handle probes that find active resources"
	do_test bnc-515172 "Location constraint with multiple expressions"
	do_test colocate-primitive-with-clone "Optional colocation with a clone"

	echo ""
	do_test systemhealth1 "System Health () #1"
	do_test systemhealth2 "System Health () #2"
	do_test systemhealth3 "System Health () #3"
	do_test systemhealthn1 "System Health (None) #1"
	do_test systemhealthn2 "System Health (None) #2"
	do_test systemhealthn3 "System Health (None) #3"
	do_test systemhealthm1 "System Health (Migrate On Red) #1"
	do_test systemhealthm2 "System Health (Migrate On Red) #2"
	do_test systemhealthm3 "System Health (Migrate On Red) #3"
	do_test systemhealtho1 "System Health (Only Green) #1"
	do_test systemhealtho2 "System Health (Only Green) #2"
	do_test systemhealtho3 "System Health (Only Green) #3"
	do_test systemhealthp1 "System Health (Progessive) #1"
	do_test systemhealthp2 "System Health (Progessive) #2"
	do_test systemhealthp3 "System Health (Progessive) #3"

	echo ""

	test_results
	diff --git a/pengine/test10/bug-lf-2361.dot b/pengine/test10/bug-lf-2361.dot
	new file mode 100644
	index 0000000000..55226586a0
	--- /dev/null
	+++ b/pengine/test10/bug-lf-2361.dot
	@@ -0,0 +1,35 @@
	+digraph "g" {
	+"cl_dummy2_running_0" [ style=dashed color="red" fontcolor="orange" ]
	+"cl_dummy2_start_0" -> "cl_dummy2_running_0" [ style = dashed]
	+"cl_dummy2_start_0" -> "dummy2:0_start_0 alice.demo" [ style = dashed]
	+"cl_dummy2_start_0" -> "dummy2:1_start_0 bob.demo" [ style = dashed]
	+"cl_dummy2_start_0" [ style=dashed color="red" fontcolor="orange" ]
	+"dummy2:0_start_0 alice.demo" -> "cl_dummy2_running_0" [ style = dashed]
	+"dummy2:0_start_0 alice.demo" [ style=dashed color="red" fontcolor="black" ]
	+"dummy2:1_start_0 bob.demo" -> "cl_dummy2_running_0" [ style = dashed]
	+"dummy2:1_start_0 bob.demo" [ style=dashed color="red" fontcolor="black" ]
	+"ms_stateful_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange" ]
	+"ms_stateful_confirmed-pre_notify_start_0" -> "ms_stateful_post_notify_running_0" [ style = bold]
	+"ms_stateful_confirmed-pre_notify_start_0" -> "ms_stateful_start_0" [ style = bold]
	+"ms_stateful_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
	+"ms_stateful_post_notify_running_0" -> "ms_stateful_confirmed-post_notify_running_0" [ style = bold]
	+"ms_stateful_post_notify_running_0" -> "stateful:0_post_notify_start_0 alice.demo" [ style = bold]
	+"ms_stateful_post_notify_running_0" -> "stateful:1_post_notify_start_0 bob.demo" [ style = bold]
	+"ms_stateful_post_notify_running_0" [ style=bold color="green" fontcolor="orange" ]
	+"ms_stateful_pre_notify_start_0" -> "ms_stateful_confirmed-pre_notify_start_0" [ style = bold]
	+"ms_stateful_pre_notify_start_0" [ style=bold color="green" fontcolor="orange" ]
	+"ms_stateful_running_0" -> "ms_stateful_post_notify_running_0" [ style = bold]
	+"ms_stateful_running_0" [ style=bold color="green" fontcolor="orange" ]
	+"ms_stateful_start_0" -> "ms_stateful_running_0" [ style = bold]
	+"ms_stateful_start_0" -> "stateful:0_start_0 alice.demo" [ style = bold]
	+"ms_stateful_start_0" -> "stateful:1_start_0 bob.demo" [ style = bold]
	+"ms_stateful_start_0" [ style=bold color="green" fontcolor="orange" ]
	+"stateful:0_post_notify_start_0 alice.demo" -> "ms_stateful_confirmed-post_notify_running_0" [ style = bold]
	+"stateful:0_post_notify_start_0 alice.demo" [ style=bold color="green" fontcolor="black" ]
	+"stateful:0_start_0 alice.demo" -> "ms_stateful_running_0" [ style = bold]
	+"stateful:0_start_0 alice.demo" [ style=bold color="green" fontcolor="black" ]
	+"stateful:1_post_notify_start_0 bob.demo" -> "ms_stateful_confirmed-post_notify_running_0" [ style = bold]
	+"stateful:1_post_notify_start_0 bob.demo" [ style=bold color="green" fontcolor="black" ]
	+"stateful:1_start_0 bob.demo" -> "ms_stateful_running_0" [ style = bold]
	+"stateful:1_start_0 bob.demo" [ style=bold color="green" fontcolor="black" ]
	+}
	diff --git a/pengine/test10/bug-lf-2361.exp b/pengine/test10/bug-lf-2361.exp
	new file mode 100644
	index 0000000000..52aad234be
	--- /dev/null
	+++ b/pengine/test10/bug-lf-2361.exp
	@@ -0,0 +1,138 @@
	+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" batch-limit="30" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <rsc_op id="5" operation="start" operation_key="stateful:0_start_0" on_node="alice.demo" on_node_uuid="df0d4306-9cf3-4c5c-a23a-027ac36da131">
	+ <primitive id="stateful:0" long-id="ms_stateful:stateful:0" class="ocf" provider="heartbeat" type="Stateful"/>
	+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="stateful:0 stateful:1 " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="stateful:0 stateful:1 " CRM_meta_notify_start_uname="alice.demo bob.demo " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="start" operation_key="ms_stateful_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="1" priority="1000000">
	+ <action_set>
	+ <rsc_op id="52" operation="notify" operation_key="stateful:0_post_notify_start_0" on_node="alice.demo" on_node_uuid="df0d4306-9cf3-4c5c-a23a-027ac36da131">
	+ <primitive id="stateful:0" long-id="ms_stateful:stateful:0" class="ocf" provider="heartbeat" type="Stateful"/>
	+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="stateful:0 stateful:1 " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="stateful:0 stateful:1 " CRM_meta_notify_start_uname="alice.demo bob.demo " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="11" operation="notify" operation_key="ms_stateful_post_notify_running_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="6" operation="start" operation_key="stateful:1_start_0" on_node="bob.demo" on_node_uuid="0af1add7-22b5-4342-9816-67e6351605de">
	+ <primitive id="stateful:1" long-id="ms_stateful:stateful:1" class="ocf" provider="heartbeat" type="Stateful"/>
	+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="stateful:0 stateful:1 " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="stateful:0 stateful:1 " CRM_meta_notify_start_uname="alice.demo bob.demo " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="7" operation="start" operation_key="ms_stateful_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3" priority="1000000">
	+ <action_set>
	+ <rsc_op id="53" operation="notify" operation_key="stateful:1_post_notify_start_0" on_node="bob.demo" on_node_uuid="0af1add7-22b5-4342-9816-67e6351605de">
	+ <primitive id="stateful:1" long-id="ms_stateful:stateful:1" class="ocf" provider="heartbeat" type="Stateful"/>
	+ <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="stateful:0 stateful:1 " CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="stateful:0 stateful:1 " CRM_meta_notify_start_uname="alice.demo bob.demo " CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="11" operation="notify" operation_key="ms_stateful_post_notify_running_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="4">
	+ <action_set>
	+ <pseudo_event id="7" operation="start" operation_key="ms_stateful_start_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="10" operation="notified" operation_key="ms_stateful_confirmed-pre_notify_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="5" priority="1000000">
	+ <action_set>
	+ <pseudo_event id="8" operation="running" operation_key="ms_stateful_running_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="5" operation="start" operation_key="stateful:0_start_0" on_node="alice.demo" on_node_uuid="df0d4306-9cf3-4c5c-a23a-027ac36da131"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="6" operation="start" operation_key="stateful:1_start_0" on_node="bob.demo" on_node_uuid="0af1add7-22b5-4342-9816-67e6351605de"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="7" operation="start" operation_key="ms_stateful_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="6">
	+ <action_set>
	+ <pseudo_event id="9" operation="notify" operation_key="ms_stateful_pre_notify_start_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="7">
	+ <action_set>
	+ <pseudo_event id="10" operation="notified" operation_key="ms_stateful_confirmed-pre_notify_start_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="9" operation="notify" operation_key="ms_stateful_pre_notify_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="8" priority="1000000">
	+ <action_set>
	+ <pseudo_event id="11" operation="notify" operation_key="ms_stateful_post_notify_running_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="8" operation="running" operation_key="ms_stateful_running_0"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="10" operation="notified" operation_key="ms_stateful_confirmed-pre_notify_start_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="9" priority="1000000">
	+ <action_set>
	+ <pseudo_event id="12" operation="notified" operation_key="ms_stateful_confirmed-post_notify_running_0">
	+ <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" crm_feature_set="3.0.1"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="11" operation="notify" operation_key="ms_stateful_post_notify_running_0"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="52" operation="notify" operation_key="stateful:0_post_notify_start_0" on_node="alice.demo" on_node_uuid="df0d4306-9cf3-4c5c-a23a-027ac36da131"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="53" operation="notify" operation_key="stateful:1_post_notify_start_0" on_node="bob.demo" on_node_uuid="0af1add7-22b5-4342-9816-67e6351605de"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+</transition_graph>
	+
	diff --git a/pengine/test10/bug-lf-2361.scores b/pengine/test10/bug-lf-2361.scores
	new file mode 100644
	index 0000000000..7f0afebf9b
	--- /dev/null
	+++ b/pengine/test10/bug-lf-2361.scores
	@@ -0,0 +1,27 @@
	+Allocation scores:
	+clone_color: ms_stateful allocation score on alice.demo: 0
	+clone_color: ms_stateful allocation score on bob.demo: 0
	+clone_color: stateful:0 allocation score on alice.demo: 0
	+clone_color: stateful:0 allocation score on bob.demo: 0
	+clone_color: stateful:1 allocation score on alice.demo: 0
	+clone_color: stateful:1 allocation score on bob.demo: 0
	+native_color: stateful:0 allocation score on alice.demo: 0
	+native_color: stateful:0 allocation score on bob.demo: 0
	+native_color: stateful:1 allocation score on alice.demo: -1000000
	+native_color: stateful:1 allocation score on bob.demo: 0
	+stateful:0 promotion score on alice.demo: -1
	+stateful:1 promotion score on bob.demo: -1
	+native_color: dummy1 allocation score on alice.demo: -1000000
	+native_color: dummy1 allocation score on bob.demo: -1000000
	+stateful:0 promotion score on alice.demo: -1000000
	+stateful:1 promotion score on bob.demo: -1000000
	+clone_color: cl_dummy2 allocation score on alice.demo: 0
	+clone_color: cl_dummy2 allocation score on bob.demo: 0
	+clone_color: dummy2:0 allocation score on alice.demo: 0
	+clone_color: dummy2:0 allocation score on bob.demo: 0
	+clone_color: dummy2:1 allocation score on alice.demo: 0
	+clone_color: dummy2:1 allocation score on bob.demo: 0
	+native_color: dummy2:0 allocation score on alice.demo: 0
	+native_color: dummy2:0 allocation score on bob.demo: 0
	+native_color: dummy2:1 allocation score on alice.demo: -1000000
	+native_color: dummy2:1 allocation score on bob.demo: 0
	diff --git a/pengine/test10/bug-lf-2361.xml b/pengine/test10/bug-lf-2361.xml
	new file mode 100644
	index 0000000000..bbdcd80294
	--- /dev/null
	+++ b/pengine/test10/bug-lf-2361.xml
	@@ -0,0 +1,114 @@
	+<cib validate-with="pacemaker-1.0" crm_feature_set="3.0.1" have-quorum="1" admin_epoch="0" epoch="167" num_updates="5" cib-last-written="Fri Feb 26 11:15:53 2010" dc-uuid="0af1add7-22b5-4342-9816-67e6351605de">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
	+ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.0.7-54d7869bfe3691eb723b1d47810e5585d8246b58"/>
	+ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="Heartbeat"/>
	+ <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1267177413"/>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="df0d4306-9cf3-4c5c-a23a-027ac36da131" type="normal" uname="alice.demo"/>
	+ <node id="0af1add7-22b5-4342-9816-67e6351605de" type="normal" uname="bob.demo"/>
	+ </nodes>
	+ <resources>
	+ <primitive class="ocf" id="dummy1" provider="heartbeat" type="Dummy"/>
	+ <master id="ms_stateful">
	+ <meta_attributes id="ms-stateful-meta_attributes">
	+ <nvpair id="ms-stateful-meta_attributes-master-max" name="master-max" value="1"/>
	+ <nvpair id="ms-stateful-meta_attributes-master-node-max" name="master-node-max" value="1"/>
	+ <nvpair id="ms-stateful-meta_attributes-clone-max" name="clone-max" value="2"/>
	+ <nvpair id="ms-stateful-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
	+ <nvpair id="ms-stateful-meta_attributes-notify" name="notify" value="true"/>
	+ </meta_attributes>
	+ <primitive class="ocf" id="stateful" provider="heartbeat" type="Stateful"/>
	+ </master>
	+ <clone id="cl_dummy2">
	+ <meta_attributes id="cl_dummy2-meta_attributes">
	+ <nvpair id="cl_dummy2-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
	+ </meta_attributes>
	+ <primitive class="ocf" id="dummy2" provider="heartbeat" type="Dummy"/>
	+ </clone>
	+ </resources>
	+ <constraints>
	+ <rsc_order first="dummy1" id="o_dummy1_before_dummy2" score="INFINITY" then="cl_dummy2"/>
	+ <rsc_order first="ms_stateful" first-action="promote" id="o_stateful_before_dummy1" score="INFINITY" then="dummy1" then-action="start"/>
	+ <rsc_colocation id="c_dummy1_on_master" rsc="dummy1" score="INFINITY" with-rsc="ms_stateful" with-rsc-role="Master"/>
	+ </constraints>
	+ <rsc_defaults/>
	+ <op_defaults/>
	+ </configuration>
	+ <status>
	+ <node_state id="0af1add7-22b5-4342-9816-67e6351605de" uname="bob.demo" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_state_transition" shutdown="0">
	+ <transient_attributes id="0af1add7-22b5-4342-9816-67e6351605de">
	+ <instance_attributes id="status-0af1add7-22b5-4342-9816-67e6351605de">
	+ <nvpair id="status-0af1add7-22b5-4342-9816-67e6351605de-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="0af1add7-22b5-4342-9816-67e6351605de">
	+ <lrm_resources>
	+ <lrm_resource id="dummy1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="dummy1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="7:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;7:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="15" rc-code="7" op-status="0" interval="0" last-run="1267178546" last-rc-change="1267178546" exec-time="30" queue-time="10" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="service2:0" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="service2:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="10:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;10:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1267177565" last-rc-change="1267177565" exec-time="70" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="service2:1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="service2:1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="32:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;32:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1267177663" last-rc-change="1267177663" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="service2:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="31:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;31:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="13" rc-code="0" op-status="0" interval="0" last-run="1267178372" last-rc-change="1267178372" exec-time="70" queue-time="10" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="service1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="service1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="9:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;9:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1267177565" last-rc-change="1267177565" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="dummy2:0" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="dummy2:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="8:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;8:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="16" rc-code="7" op-status="0" interval="0" last-run="1267178546" last-rc-change="1267178546" exec-time="60" queue-time="10" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="stateful:0" type="Stateful" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="stateful:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="8:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;8:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1267177565" last-rc-change="1267177565" exec-time="20" queue-time="10" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="stateful:1" type="Stateful" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="stateful:1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="6:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;6:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1267177663" last-rc-change="1267177663" exec-time="150" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:1_pre_notify_stop_0" operation="notify" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="50:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;50:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1267178372" last-rc-change="1267178372" exec-time="190" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="6:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;6:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1267178374" last-rc-change="1267178374" exec-time="80" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state id="df0d4306-9cf3-4c5c-a23a-027ac36da131" uname="alice.demo" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_state_transition" shutdown="0">
	+ <transient_attributes id="df0d4306-9cf3-4c5c-a23a-027ac36da131">
	+ <instance_attributes id="status-df0d4306-9cf3-4c5c-a23a-027ac36da131">
	+ <nvpair id="status-df0d4306-9cf3-4c5c-a23a-027ac36da131-probe_complete" name="probe_complete" value="true"/>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="df0d4306-9cf3-4c5c-a23a-027ac36da131">
	+ <lrm_resources>
	+ <lrm_resource id="dummy1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="dummy1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="4:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;4:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="20" rc-code="7" op-status="0" interval="0" last-run="1267178547" last-rc-change="1267178547" exec-time="70" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="service2:0" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="service2:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="6:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;6:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1267177566" last-rc-change="1267177566" exec-time="90" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="service2:0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="38:5:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;38:5:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="12" rc-code="0" op-status="0" interval="0" last-run="1267177672" last-rc-change="1267177672" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="service2:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="37:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;37:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1267178367" last-rc-change="1267178367" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="service1" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="service1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="5:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;5:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="3" rc-code="7" op-status="0" interval="0" last-run="1267177566" last-rc-change="1267177566" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="service1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="36:5:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;36:5:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="11" rc-code="0" op-status="0" interval="0" last-run="1267177671" last-rc-change="1267177671" exec-time="60" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="service1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="35:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;35:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1267178368" last-rc-change="1267178368" exec-time="30" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="dummy2:0" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="dummy2:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="5:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;5:15:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="21" rc-code="7" op-status="0" interval="0" last-run="1267178547" last-rc-change="1267178547" exec-time="140" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="stateful:0" type="Stateful" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="stateful:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="4:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:7;4:0:7:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1267177566" last-rc-change="1267177566" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:0_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="5:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;5:3:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1267177664" last-rc-change="1267177664" exec-time="190" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:0_promote_0" operation="promote" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="8:4:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;8:4:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1267177668" last-rc-change="1267177668" exec-time="100" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:0_demote_0" operation="demote" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="5:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;5:6:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="16" rc-code="0" op-status="0" interval="0" last-run="1267178370" last-rc-change="1267178370" exec-time="120" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:0_pre_notify_stop_0" operation="notify" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="49:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;49:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1267178373" last-rc-change="1267178373" exec-time="50" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="stateful:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.1" transition-key="5:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" transition-magic="0:0;5:7:0:2b2cc77a-01ad-4266-87e0-29d049ead5cc" call-id="19" rc-code="0" op-status="0" interval="0" last-run="1267178375" last-rc-change="1267178375" exec-time="140" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ </status>
	+</cib>

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 5:54 PM (1 d, 2 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002397
Default Alt Text: (93 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions