No OneTemporary
Actions

Size

122 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crm/pengine/clone.c b/crm/pengine/clone.c
	index 2c724c29f9..175ca1ba41 100644
	--- a/crm/pengine/clone.c
	+++ b/crm/pengine/clone.c
	@@ -1,1270 +1,1270 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <lha_internal.h>

	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>
	#include <lib/crm/pengine/utils.h>

	#define VARIANT_CLONE 1
	#include <lib/crm/pengine/variant.h>

	gint sort_clone_instance(gconstpointer a, gconstpointer b);

	void clone_create_notifications(
	resource_t rsc, action_t action, action_t *action_complete,
	pe_working_set_t *data_set);

	void child_stopping_constraints(
	clone_variant_data_t *clone_data,
	resource_t self, resource_t child, resource_t *last,
	pe_working_set_t *data_set);

	void child_starting_constraints(
	clone_variant_data_t *clone_data,
	resource_t self, resource_t child, resource_t *last,
	pe_working_set_t *data_set);


	static node_t *
	parent_node_instance(const resource_t rsc, node_t node)
	{
	node_t *ret = NULL;
	if(node != NULL) {
	ret = pe_find_node_id(
	rsc->parent->allowed_nodes, node->details->id);
	}
	return ret;
	}

	gint sort_clone_instance(gconstpointer a, gconstpointer b)
	{
	int level = LOG_DEBUG_3;
	node_t *node1 = NULL;
	node_t *node2 = NULL;

	gboolean can1 = TRUE;
	gboolean can2 = TRUE;

	const resource_t resource1 = (const resource_t)a;
	const resource_t resource2 = (const resource_t)b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	/* allocation order:
	* - active instances
	* - instances running on nodes with the least copies
	* - active instances on nodes that cant support them or are to be fenced
	* - failed instances
	* - inactive instances
	*/

	do_crm_log(level+1, "%s ? %s", resource1->id, resource2->id);
	if(resource1->running_on && resource2->running_on) {
	if(g_list_length(resource1->running_on) < g_list_length(resource2->running_on)) {
	do_crm_log(level, "%s < %s: running_on", resource1->id, resource2->id);
	return -1;

	} else if(g_list_length(resource1->running_on) > g_list_length(resource2->running_on)) {
	do_crm_log(level, "%s > %s: running_on", resource1->id, resource2->id);
	return 1;
	}
	}

	if(resource1->running_on) {
	node1 = resource1->running_on->data;
	}
	if(resource2->running_on) {
	node2 = resource2->running_on->data;
	}

	if(resource1->priority < resource2->priority) {
	do_crm_log(level, "%s < %s: score", resource1->id, resource2->id);
	return 1;

	} else if(resource1->priority > resource2->priority) {
	do_crm_log(level, "%s > %s: score", resource1->id, resource2->id);
	return -1;
	}

	if(node1 == NULL && node2 == NULL) {
	do_crm_log(level, "%s == %s: not active",
	resource1->id, resource2->id);
	return 0;
	}

	if(node1 != node2) {
	if(node1 == NULL) {
	do_crm_log(level, "%s > %s: active", resource1->id, resource2->id);
	return 1;
	} else if(node2 == NULL) {
	do_crm_log(level, "%s < %s: active", resource1->id, resource2->id);
	return -1;
	}
	}

	can1 = can_run_resources(node1);
	can2 = can_run_resources(node2);
	if(can1 != can2) {
	if(can1) {
	do_crm_log(level, "%s < %s: can", resource1->id, resource2->id);
	return -1;
	}
	do_crm_log(level, "%s > %s: can", resource1->id, resource2->id);
	return 1;
	}

	node1 = parent_node_instance(resource1, node1);
	node2 = parent_node_instance(resource2, node2);
	if(node1 != NULL && node2 == NULL) {
	do_crm_log(level, "%s < %s: not allowed", resource1->id, resource2->id);
	return -1;
	} else if(node1 == NULL && node2 != NULL) {
	do_crm_log(level, "%s > %s: not allowed", resource1->id, resource2->id);
	return 1;
	}

	if(node1 == NULL) {
	do_crm_log(level, "%s == %s: not allowed", resource1->id, resource2->id);
	return 0;
	}

	if(node1->count < node2->count) {
	do_crm_log(level, "%s < %s: count", resource1->id, resource2->id);
	return -1;

	} else if(node1->count > node2->count) {
	do_crm_log(level, "%s > %s: count", resource1->id, resource2->id);
	return 1;
	}

	if(node1->weight < node2->weight) {
	do_crm_log(level, "%s < %s: node score", resource1->id, resource2->id);
	return 1;

	} else if(node1->weight > node2->weight) {
	do_crm_log(level, "%s > %s: node score", resource1->id, resource2->id);
	return -1;
	}

	do_crm_log(level, "%s == %s: default %d", resource1->id, resource2->id, node2->weight);
	return 0;
	}

	static node_t *
	can_run_instance(resource_t rsc, node_t node)
	{
	node_t *local_node = NULL;
	clone_variant_data_t *clone_data = NULL;
	if(can_run_resources(node) == FALSE) {
	goto bail;
	}

	local_node = parent_node_instance(rsc, node);
	get_clone_variant_data(clone_data, rsc->parent);

	if(local_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed",
	rsc->id, node->details->uname);
	goto bail;

	} else if(local_node->count < clone_data->clone_node_max) {
	return local_node;

	} else {
	crm_debug_2("%s cannot run on %s: node full",
	rsc->id, node->details->uname);
	}

	bail:
	if(node) {
	common_update_score(rsc, node->details->id, -INFINITY);
	}
	return NULL;
	}


	static node_t *
	color_instance(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *local_node = NULL;
	node_t *chosen = NULL;

	crm_debug_2("Processing %s", rsc->id);

	if(rsc->provisional == FALSE) {
	return rsc->allocated_to;

	} else if(rsc->is_allocating) {
	crm_debug("Dependancy loop detected involving %s", rsc->id);
	return NULL;
	}

	if(rsc->allowed_nodes) {
	slist_iter(try_node, node_t, rsc->allowed_nodes, lpc,
	can_run_instance(rsc, try_node);
	);
	}

	chosen = rsc->cmds->color(rsc, data_set);
	if(chosen) {
	local_node = pe_find_node_id(
	rsc->parent->allowed_nodes, chosen->details->id);

	if(local_node) {
	local_node->count++;
	} else if(rsc->is_managed) {
	/* what to do? we can't enforce per-node limits in this case */
	crm_config_err("%s not found in %s (list=%d)",
	chosen->details->id, rsc->parent->id,
	g_list_length(rsc->parent->allowed_nodes));
	}
	}

	return chosen;
	}

	static void append_parent_colocation(resource_t rsc, resource_t child)
	{
	slist_iter(cons, rsc_colocation_t, rsc->rsc_cons, lpc,
	child->rsc_cons = g_list_append(child->rsc_cons, cons));
	slist_iter(cons, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
	child->rsc_cons_lhs = g_list_append(child->rsc_cons_lhs, cons));
	}

	node_t *
	clone_color(resource_t rsc, pe_working_set_t data_set)
	{
	int allocated = 0;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	if(rsc->provisional == FALSE) {
	return NULL;

	} else if(rsc->is_allocating) {
	crm_debug("Dependancy loop detected involving %s", rsc->id);
	return NULL;
	}

	rsc->is_allocating = TRUE;
	crm_debug_2("Processing %s", rsc->id);

	/* this information is used by sort_clone_instance() when deciding in which
	* order to allocate clone instances
	*/
	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,

	rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
	constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->score/INFINITY, TRUE);
	);

	dump_node_scores(LOG_DEBUG_2, rsc, __FUNCTION__, rsc->allowed_nodes);

	/* count now tracks the number of clones currently allocated */
	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
	node->count = 0;
	);

	slist_iter(child, resource_t, rsc->children, lpc,
	if(g_list_length(child->running_on) > 0) {
	node_t *child_node = child->running_on->data;
	node_t *local_node = parent_node_instance(
	child, child->running_on->data);
	if(local_node) {
	local_node->count++;
	} else {
	crm_err("%s is running on %s which isn't allowed",
	child->id, child_node->details->uname);
	}
	}
	);

	rsc->children = g_list_sort(rsc->children, sort_clone_instance);

	/* count now tracks the number of clones we have allocated */
	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
	node->count = 0;
	);

	rsc->allowed_nodes = g_list_sort(
	rsc->allowed_nodes, sort_node_weight);


	slist_iter(child, resource_t, rsc->children, lpc,
	if(allocated >= clone_data->clone_max) {
	crm_debug("Child %s not allocated - limit reached", child->id);
	resource_location(child, NULL, -INFINITY, "clone_color:limit_reached", data_set);
	} else {
	append_parent_colocation(rsc, child);
	}

	if(color_instance(child, data_set)) {
	allocated++;
	}
	);

	crm_debug("Allocated %d %s instances of a possible %d",
	allocated, rsc->id, clone_data->clone_max);

	rsc->provisional = FALSE;
	rsc->is_allocating = FALSE;

	return NULL;
	}

	static void
	clone_update_pseudo_status(
	resource_t child, gboolean stopping, gboolean *starting)
	{
	CRM_ASSERT(stopping != NULL);
	CRM_ASSERT(starting != NULL);

	slist_iter(
	action, action_t, child->actions, lpc,

	if(starting && stopping) {
	return;

	} else if(action->optional) {
	crm_debug_3("Skipping optional: %s", action->uuid);
	continue;

	} else if(action->pseudo == FALSE && action->runnable == FALSE){
	crm_debug_3("Skipping unrunnable: %s", action->uuid);
	continue;

	} else if(safe_str_eq(CRMD_ACTION_STOP, action->task)) {
	crm_debug_2("Stopping due to: %s", action->uuid);
	*stopping = TRUE;

	} else if(safe_str_eq(CRMD_ACTION_START, action->task)) {
	if(action->runnable == FALSE) {
	crm_debug_3("Skipping pseudo-op: %s run=%d, pseudo=%d",
	action->uuid, action->runnable, action->pseudo);
	} else {
	crm_debug_2("Starting due to: %s", action->uuid);
	crm_debug_3("%s run=%d, pseudo=%d",
	action->uuid, action->runnable, action->pseudo);
	*starting = TRUE;
	}
	}
	);

	}

	void clone_create_actions(resource_t rsc, pe_working_set_t data_set)
	{
	gboolean child_starting = FALSE;
	gboolean child_stopping = FALSE;
	action_t *stop = NULL;
	action_t *start = NULL;
	action_t *action_complete = NULL;
	resource_t *last_start_rsc = NULL;
	resource_t *last_stop_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	crm_debug_2("Creating actions for %s", rsc->id);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->create_actions(child_rsc, data_set);
	clone_update_pseudo_status(
	child_rsc, &child_stopping, &child_starting);

	if(child_rsc->starting) {
	last_start_rsc = child_rsc;
	}
	if(child_rsc->stopping) {
	last_stop_rsc = child_rsc;
	}
	);

	/* start */
	start = start_action(rsc, NULL, !child_starting);
	action_complete = custom_action(
	rsc, started_key(rsc),
	CRMD_ACTION_STARTED, NULL, !child_starting, TRUE, data_set);

	start->pseudo = TRUE;
	start->runnable = TRUE;
	action_complete->pseudo = TRUE;
	action_complete->runnable = TRUE;
	action_complete->priority = INFINITY;
	/* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */

	child_starting_constraints(clone_data, rsc, NULL, last_start_rsc, data_set);

	clone_create_notifications(
	rsc, start, action_complete, data_set);


	/* stop */
	stop = stop_action(rsc, NULL, !child_stopping);
	action_complete = custom_action(
	rsc, stopped_key(rsc),
	CRMD_ACTION_STOPPED, NULL, !child_stopping, TRUE, data_set);

	stop->pseudo = TRUE;
	stop->runnable = TRUE;
	action_complete->pseudo = TRUE;
	action_complete->runnable = TRUE;
	action_complete->priority = INFINITY;
	/* crm_err("Upgrading priority for %s to INFINITY", action_complete->uuid); */

	child_stopping_constraints(clone_data, rsc, NULL, last_stop_rsc, data_set);


	clone_create_notifications(rsc, stop, action_complete, data_set);
	rsc->actions = rsc->actions;

	if(stop->post_notified != NULL && start->pre_notify != NULL) {
	order_actions(stop->post_notified, start->pre_notify, pe_order_optional);
	}
	}

	void
	clone_create_notifications(
	resource_t rsc, action_t action, action_t *action_complete,
	pe_working_set_t *data_set)
	{
	/*
	* pre_notify -> pre_notify_complete -> pseudo_action
	* -> (real actions) -> pseudo_action_complete
	* -> post_notify -> post_notify_complete
	*
	* if the pre_noitfy requires confirmation,
	* then a list of confirmations will be added as triggers
	* to pseudo_action in clone_expand()
	*/
	action_t *notify = NULL;
	action_t *notify_complete = NULL;
	enum action_tasks task;
	char *notify_key = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	if(rsc->notify == FALSE) {
	return;
	}

	task = text2task(action->task);

	/* create pre_notify */
	notify_key = generate_notify_key(
	rsc->id, "pre", action->task);
	notify = custom_action(rsc, notify_key,
	CRMD_ACTION_NOTIFY, NULL,
	action->optional, TRUE, data_set);

	add_hash_param(notify->meta, "notify_type", "pre");
	add_hash_param(notify->meta, "notify_operation", action->task);
	if(clone_data->notify_confirm) {
	add_hash_param(notify->meta, "notify_confirm", "yes");
	} else {
	add_hash_param(notify->meta, "notify_confirm", "no");
	}

	/* create pre_notify_complete */
	notify_key = generate_notify_key(
	rsc->id, "confirmed-pre", action->task);
	notify_complete = custom_action(rsc, notify_key,
	CRMD_ACTION_NOTIFIED, NULL,
	action->optional, TRUE, data_set);
	add_hash_param(notify_complete->meta, "notify_type", "pre");
	add_hash_param(notify_complete->meta, "notify_operation", action->task);
	if(clone_data->notify_confirm) {
	add_hash_param(notify->meta, "notify_confirm", "yes");
	} else {
	add_hash_param(notify->meta, "notify_confirm", "no");
	}
	notify->pseudo = TRUE;
	notify->runnable = TRUE;
	notify_complete->pseudo = TRUE;
	notify_complete->runnable = TRUE;

	/* pre_notify before pre_notify_complete */
	custom_action_order(
	rsc, NULL, notify,
	rsc, NULL, notify_complete,
	pe_order_implies_left, data_set);

	/* pre_notify_complete before action */
	custom_action_order(
	rsc, NULL, notify_complete,
	rsc, NULL, action,
	pe_order_implies_left, data_set);

	action->pre_notify = notify;
	action->pre_notified = notify_complete;

	/* create post_notify */
	notify_key = generate_notify_key
	(rsc->id, "post", action->task);
	notify = custom_action(rsc, notify_key,
	CRMD_ACTION_NOTIFY, NULL,
	action_complete->optional, TRUE, data_set);
	add_hash_param(notify->meta, "notify_type", "post");
	add_hash_param(notify->meta, "notify_operation", action->task);
	if(clone_data->notify_confirm) {
	add_hash_param(notify->meta, "notify_confirm", "yes");
	} else {
	add_hash_param(notify->meta, "notify_confirm", "no");
	}

	/* action_complete before post_notify */
	custom_action_order(
	rsc, NULL, action_complete,
	rsc, NULL, notify,
	pe_order_implies_right, data_set);

	/* create post_notify_complete */
	notify_key = generate_notify_key(
	rsc->id, "confirmed-post", action->task);
	notify_complete = custom_action(rsc, notify_key,
	CRMD_ACTION_NOTIFIED, NULL,
	action->optional, TRUE, data_set);
	add_hash_param(notify_complete->meta, "notify_type", "pre");
	add_hash_param(notify_complete->meta, "notify_operation", action->task);
	if(clone_data->notify_confirm) {
	add_hash_param(notify->meta, "notify_confirm", "yes");
	} else {
	add_hash_param(notify->meta, "notify_confirm", "no");
	}

	notify->pseudo = TRUE;
	notify->runnable = TRUE;
	notify->priority = INFINITY;
	notify->runnable = action_complete->runnable;

	notify_complete->pseudo = TRUE;
	notify_complete->runnable = TRUE;
	notify_complete->priority = INFINITY;
	notify_complete->runnable = action_complete->runnable;

	/* post_notify before post_notify_complete */
	custom_action_order(
	rsc, NULL, notify,
	rsc, NULL, notify_complete,
	pe_order_implies_left, data_set);

	action->post_notify = notify;
	action->post_notified = notify_complete;


	if(safe_str_eq(action->task, CRMD_ACTION_STOP)) {
	/* post_notify_complete before start */
	custom_action_order(
	rsc, NULL, notify_complete,
	rsc, start_key(rsc), NULL,
	pe_order_optional, data_set);

	} else if(safe_str_eq(action->task, CRMD_ACTION_START)) {
	/* post_notify_complete before promote */
	custom_action_order(
	rsc, NULL, notify_complete,
	rsc, promote_key(rsc), NULL,
	pe_order_optional, data_set);

	} else if(safe_str_eq(action->task, CRMD_ACTION_DEMOTE)) {
	/* post_notify_complete before promote */
	custom_action_order(
	rsc, NULL, notify_complete,
	rsc, stop_key(rsc), NULL,
	pe_order_optional, data_set);
	}
	}

	void
	child_starting_constraints(
	clone_variant_data_t *clone_data,
	resource_t rsc, resource_t child, resource_t *last,
	pe_working_set_t *data_set)
	{
	if(child == NULL && last == NULL) {
	crm_debug("%s has no active children", rsc->id);
	return;
	}

	if(child != NULL) {
	order_start_start(rsc, child, pe_order_runnable_left);

	custom_action_order(
	child, start_key(child), NULL,
	rsc, started_key(rsc), NULL,
	pe_order_optional, data_set);
	}

	if(clone_data->ordered) {
	if(child == NULL) {
	/* last child start before global started */
	custom_action_order(
	last, start_key(last), NULL,
	rsc, started_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	} else if(last == NULL) {
	/* global start before first child start */
	order_start_start(
	rsc, child, pe_order_implies_left);

	} else {
	/* child/child relative start */
	order_start_start(last, child, pe_order_implies_left);
	}
	}
	}

	void
	child_stopping_constraints(
	clone_variant_data_t *clone_data,
	resource_t rsc, resource_t child, resource_t *last,
	pe_working_set_t *data_set)
	{
	if(child == NULL && last == NULL) {
	crm_debug("%s has no active children", rsc->id);
	return;
	}

	if(child != NULL) {
	- order_stop_stop(rsc, child, pe_order_optional);
	+ order_stop_stop(rsc, child, pe_order_shutdown);

	custom_action_order(
	child, stop_key(child), NULL,
	rsc, stopped_key(rsc), NULL,
	pe_order_optional, data_set);
	}

	if(clone_data->ordered) {
	if(last == NULL) {
	/* first child stop before global stopped */
	custom_action_order(
	child, stop_key(child), NULL,
	rsc, stopped_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	} else if(child == NULL) {
	/* global stop before last child stop */
	order_stop_stop(
	rsc, last, pe_order_implies_left);
	} else {
	/* child/child relative stop */
	order_stop_stop(child, last, pe_order_implies_left);
	}
	}
	}


	void
	clone_internal_constraints(resource_t rsc, pe_working_set_t data_set)
	{
	resource_t *last_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	native_internal_constraints(rsc, data_set);

	/* global stop before stopped */
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	rsc, stopped_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	/* global start before started */
	custom_action_order(
	rsc, start_key(rsc), NULL,
	rsc, started_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	/* global stopped before start */
	custom_action_order(
	rsc, stopped_key(rsc), NULL,
	rsc, start_key(rsc), NULL,
	pe_order_optional, data_set);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	child_starting_constraints(
	clone_data, rsc, child_rsc, last_rsc, data_set);

	child_stopping_constraints(
	clone_data, rsc, child_rsc, last_rsc, data_set);

	last_rsc = child_rsc;
	);
	}

	static resource_t*
	find_compatible_child(resource_t local_child, resource_t rsc)
	{
	node_t *local_node = NULL;
	node_t *node = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	local_node = local_child->allocated_to;
	if(local_node == NULL) {
	crm_debug("Can't colocate unrunnable child %s with %s",
	local_child->id, rsc->id);
	return NULL;
	}

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	node = child_rsc->allocated_to;
	if(node->details == local_node->details) {
	crm_info("Colocating %s with %s on %s",
	local_child->id, child_rsc->id, node->details->uname);
	return child_rsc;
	}
	);
	crm_debug("Can't colocate child %s with %s",
	local_child->id, rsc->id);
	return NULL;
	}

	void clone_rsc_colocation_lh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	gboolean do_interleave = FALSE;
	resource_t *rsc = constraint->rsc_lh;
	clone_variant_data_t *clone_data = NULL;
	clone_variant_data_t *clone_data_rh = NULL;

	if(rsc == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if(constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;

	} else {
	crm_debug_4("Processing constraints from %s", rsc->id);
	}

	get_clone_variant_data(clone_data, rsc);

	if(constraint->rsc_rh->variant == pe_clone) {
	get_clone_variant_data(
	clone_data_rh, constraint->rsc_rh);
	if(clone_data->clone_node_max
	!= clone_data_rh->clone_node_max) {
	crm_config_err("Cannot interleave "XML_CIB_TAG_INCARNATION
	" %s and %s because"
	" they do not support the same number of"
	" resources per node",
	constraint->rsc_lh->id, constraint->rsc_rh->id);

	/* only the LHS side needs to be labeled as interleave */
	} else if(clone_data->interleave) {
	do_interleave = TRUE;

	} else if(constraint->score >= INFINITY) {
	GListPtr lhs = NULL, rhs = NULL;
	lhs = rsc_lh->allowed_nodes;

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,
	if(child_rsc->allocated_to != NULL) {
	rhs = g_list_append(rhs, child_rsc->allocated_to);
	}
	);

	rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);

	pe_free_shallow_adv(rhs, FALSE);
	pe_free_shallow(lhs);
	return;
	}

	} else if(constraint->score >= INFINITY) {
	crm_config_err("Manditory co-location of clones (%s) with other"
	" non-clone (%s) resources is not supported",
	rsc_lh->id, rsc_rh->id);
	return;
	}

	if(do_interleave) {
	resource_t *rh_child = NULL;

	slist_iter(lh_child, resource_t, rsc->children, lpc,

	CRM_ASSERT(lh_child != NULL);
	rh_child = find_compatible_child(lh_child, rsc_rh);
	if(rh_child == NULL) {
	continue;
	}
	lh_child->cmds->rsc_colocation_lh(
	lh_child, rh_child, constraint);
	);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->rsc_colocation_lh(child_rsc, constraint->rsc_rh, constraint);
	);
	}

	void clone_rsc_colocation_rh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	clone_variant_data_t *clone_data = NULL;
	CRM_CHECK(rsc_lh != NULL, return);
	CRM_CHECK(rsc_lh->variant == pe_native, return);

	get_clone_variant_data(clone_data, rsc_rh);

	crm_debug_3("Processing constraint %s: %d", constraint->id, constraint->score);

	if(rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;

	} else if(rsc_rh->provisional) {
	crm_debug_3("%s is still provisional", rsc_rh->id);
	return;

	} else if(constraint->score >= INFINITY) {
	GListPtr lhs = NULL, rhs = NULL;
	lhs = rsc_lh->allowed_nodes;

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,
	if(child_rsc->allocated_to != NULL) {
	rhs = g_list_append(rhs, child_rsc->allocated_to);
	}
	);

	rsc_lh->allowed_nodes = node_list_and(lhs, rhs, FALSE);

	pe_free_shallow_adv(rhs, FALSE);
	pe_free_shallow(lhs);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,

	child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
	);
	}

	void clone_rsc_order_lh(resource_t rsc, order_constraint_t order, pe_working_set_t *data_set)
	{
	resource_t *r1 = NULL;
	resource_t *r2 = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_2("%s->%s", order->lh_action_task, order->rh_action_task);

	r1 = uber_parent(rsc);
	r2 = uber_parent(order->rh_rsc);

	if(r1 == r2) {
	native_rsc_order_lh(rsc, order, data_set);
	return;
	}

	#if 0
	if(order->type != pe_order_optional) {
	crm_debug("Upgraded ordering constraint %d - 0x%.6x", order->id, order->type);
	native_rsc_order_lh(rsc, order, data_set);
	}
	#endif

	if(order->type & pe_order_implies_left) {
	if(rsc->variant == order->rh_rsc->variant) {
	crm_debug_2("Clone-to-clone ordering: %s -> %s 0x%.6x",
	order->lh_action_task, order->rh_action_task, order->type);
	/* stop instances on the same nodes as stopping RHS instances */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	native_rsc_order_lh(child_rsc, order, data_set);
	);
	} else {
	/* stop everything */
	crm_debug_2("Clone-to-* ordering: %s -> %s 0x%.6x",
	order->lh_action_task, order->rh_action_task, order->type);
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	native_rsc_order_lh(child_rsc, order, data_set);
	);
	}
	}

	convert_non_atomic_task(rsc, order);
	native_rsc_order_lh(rsc, order, data_set);
	}

	void clone_rsc_order_rh(
	action_t lh_action, resource_t rsc, order_constraint_t *order)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_2("%s->%s", lh_action->uuid, order->rh_action_task);

	native_rsc_order_rh(lh_action, rsc, order);

	}

	void clone_rsc_location(resource_t rsc, rsc_to_node_t constraint)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_debug_3("Processing location constraint %s for %s",
	constraint->id, rsc->id);

	native_rsc_location(rsc, constraint);
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->rsc_location(child_rsc, constraint);
	);
	}

	static gint
	sort_notify_entries(gconstpointer a, gconstpointer b)
	{
	int tmp;
	const notify_entry_t *entry_a = a;
	const notify_entry_t *entry_b = b;

	if(entry_a == NULL && entry_b == NULL) { return 0; }
	if(entry_a == NULL) { return 1; }
	if(entry_b == NULL) { return -1; }

	if(entry_a->rsc == NULL && entry_b->rsc == NULL) { return 0; }
	if(entry_a->rsc == NULL) { return 1; }
	if(entry_b->rsc == NULL) { return -1; }

	tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
	if(tmp != 0) {
	return tmp;
	}

	if(entry_a->node == NULL && entry_b->node == NULL) { return 0; }
	if(entry_a->node == NULL) { return 1; }
	if(entry_b->node == NULL) { return -1; }

	return strcmp(entry_a->node->details->id, entry_b->node->details->id);
	}

	static void
	expand_list(GListPtr list, int clones,
	char rsc_list, char node_list, char **uuid_list)
	{
	const char *uname = NULL;
	const char *rsc_id = NULL;
	const char *last_rsc_id = NULL;

	CRM_CHECK(list != NULL, return);
	if(rsc_list) {
	CRM_CHECK(rsc_list == NULL, rsc_list = NULL);
	}
	if(node_list) {
	CRM_CHECK(node_list == NULL, node_list = NULL);
	}

	slist_iter(entry, notify_entry_t, list, lpc,

	CRM_CHECK(entry != NULL, continue);
	CRM_CHECK(entry->rsc != NULL, continue);
	CRM_CHECK(entry->node != NULL, continue);

	rsc_id = entry->rsc->id;
	uname = entry->node->details->uname;

	CRM_ASSERT(uname != NULL);
	CRM_ASSERT(rsc_id != NULL);

	/* filter dups */
	if(safe_str_eq(rsc_id, last_rsc_id)) {
	continue;
	}
	last_rsc_id = rsc_id;

	if(rsc_list != NULL) {
	int existing_len = 0;
	int len = 2 + strlen(rsc_id); /* +1 space, +1 EOS */
	if(rsc_list && *rsc_list) {
	existing_len = strlen(*rsc_list);
	}

	crm_debug_5("Adding %s (%dc) at offset %d",
	rsc_id, len-2, existing_len);
	crm_realloc(*rsc_list, len + existing_len);
	sprintf(*rsc_list + existing_len, "%s ", rsc_id);
	}

	if(node_list != NULL) {
	int existing_len = 0;
	int len = 2 + strlen(uname);
	if(node_list && *node_list) {
	existing_len = strlen(*node_list);
	}

	crm_debug_5("Adding %s (%dc) at offset %d",
	uname, len-2, existing_len);
	crm_realloc(*node_list, len + existing_len);
	sprintf(*node_list + existing_len, "%s ", uname);
	}
	);
	}

	void clone_expand(resource_t rsc, pe_working_set_t data_set)
	{
	char *rsc_list = NULL;
	char *node_list = NULL;
	char *uuid_list = NULL;

	notify_data_t *n_data = NULL;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	crm_malloc0(n_data, sizeof(notify_data_t));
	n_data->keys = g_hash_table_new_full(
	g_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	crm_debug_2("Processing actions from %s", rsc->id);


	if(rsc->notify) {
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	slist_iter(
	op, action_t, rsc->actions, lpc2,

	child_rsc->cmds->create_notify_element(
	child_rsc, op, n_data, data_set);
	);
	);
	}

	/* expand the notify data */
	if(rsc->notify && n_data->stop) {
	n_data->stop = g_list_sort(
	n_data->stop, sort_notify_entries);
	rsc_list = NULL; node_list = NULL;
	expand_list(n_data->stop, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_stop_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_stop_uname"), node_list);
	}

	if(rsc->notify && n_data->start) {
	n_data->start = g_list_sort(
	n_data->start, sort_notify_entries);
	rsc_list = NULL; node_list = NULL;
	expand_list(n_data->start, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_start_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_start_uname"), node_list);
	}

	if(rsc->notify && n_data->demote) {
	n_data->demote = g_list_sort(
	n_data->demote, sort_notify_entries);
	rsc_list = NULL; node_list = NULL;
	expand_list(n_data->demote, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_demote_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_demote_uname"), node_list);
	}

	if(rsc->notify && n_data->promote) {
	n_data->promote = g_list_sort(
	n_data->promote, sort_notify_entries);
	rsc_list = NULL; node_list = NULL; uuid_list = NULL;
	expand_list(n_data->promote, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_promote_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_promote_uname"), node_list);
	}

	if(rsc->notify && n_data->active) {
	n_data->active = g_list_sort(
	n_data->active, sort_notify_entries);
	rsc_list = NULL; node_list = NULL; uuid_list = NULL;
	expand_list(n_data->active, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_active_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_active_uname"), node_list);
	}

	if(rsc->notify && n_data->slave) {
	n_data->slave = g_list_sort(
	n_data->slave, sort_notify_entries);
	rsc_list = NULL; node_list = NULL; uuid_list = NULL;
	expand_list(n_data->slave, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_slave_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_slave_uname"), node_list);
	}

	if(rsc->notify && n_data->master) {
	n_data->master = g_list_sort(
	n_data->master, sort_notify_entries);
	rsc_list = NULL; node_list = NULL; uuid_list = NULL;
	expand_list(n_data->master, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_master_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_master_uname"), node_list);
	}

	if(rsc->notify && n_data->inactive) {
	n_data->inactive = g_list_sort(
	n_data->inactive, sort_notify_entries);
	rsc_list = NULL; node_list = NULL; uuid_list = NULL;
	expand_list(n_data->inactive, clone_data->clone_max,
	&rsc_list, &node_list, &uuid_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_inactive_resource"), rsc_list);
	g_hash_table_insert(
	n_data->keys,
	crm_strdup("notify_inactive_uname"), node_list);
	}

	/* yes, we DO need this second loop */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->expand(child_rsc, data_set);

	);

	/* slist_iter( */
	/* action, action_t, rsc->actions, lpc2, */

	/* if(safe_str_eq(action->task, CRMD_ACTION_NOTIFY)) { */
	/* action->meta_xml = notify_xml; */
	/* } */
	/* ); */

	native_expand(rsc, data_set);

	/* destroy the notify_data */
	pe_free_shallow(n_data->stop);
	pe_free_shallow(n_data->start);
	pe_free_shallow(n_data->demote);
	pe_free_shallow(n_data->promote);
	pe_free_shallow(n_data->master);
	pe_free_shallow(n_data->slave);
	pe_free_shallow(n_data->active);
	pe_free_shallow(n_data->inactive);
	g_hash_table_destroy(n_data->keys);
	crm_free(n_data);
	}


	static gint sort_rsc_id(gconstpointer a, gconstpointer b)
	{
	const resource_t resource1 = (const resource_t)a;
	const resource_t resource2 = (const resource_t)b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	return strcmp(resource1->id, resource2->id);
	}

	gboolean
	clone_create_probe(resource_t rsc, node_t node, action_t *complete,
	gboolean force, pe_working_set_t *data_set)
	{
	gboolean any_created = FALSE;
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);

	rsc->children = g_list_sort(rsc->children, sort_rsc_id);

	if(rsc->globally_unique == FALSE && clone_data->clone_node_max == 1) {
	/* only look for one copy */
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	if(pe_find_node_id(child_rsc->running_on, node->details->id)) {
	return child_rsc->cmds->create_probe(
	child_rsc, node, complete, force, data_set);
	}
	);
	}
	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	if(child_rsc->cmds->create_probe(
	child_rsc, node, complete, force, data_set)) {
	any_created = TRUE;
	}

	if(any_created
	&& rsc->globally_unique == FALSE
	&& clone_data->clone_node_max == 1) {
	/* only look for one copy (clone :0) */
	break;
	}
	);

	return any_created;
	}
	diff --git a/crm/pengine/graph.c b/crm/pengine/graph.c
	index af1d9686a6..e5e5ef0c83 100644
	--- a/crm/pengine/graph.c
	+++ b/crm/pengine/graph.c
	@@ -1,607 +1,623 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <lha_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>

	#include <glib.h>

	#include <allocate.h>
	#include <lib/crm/pengine/utils.h>
	#include <utils.h>

	gboolean update_action(action_t *action);

	gboolean
	update_action_states(GListPtr actions)
	{
	crm_debug_2("Updating %d actions", g_list_length(actions));
	slist_iter(
	action, action_t, actions, lpc,

	update_action(action);
	);

	return TRUE;
	}

	gboolean
	update_action(action_t *action)
	{
	int local_type = 0;
	int log_level = LOG_DEBUG_2;
	gboolean changed = FALSE;

	do_crm_log(log_level, "Processing action %s: %s %s %s",
	action->uuid,
	action->optional?"optional":"required",
	action->runnable?"runnable":"unrunnable",
	action->pseudo?"pseudo":action->task);

	slist_iter(
	other, action_wrapper_t, action->actions_before, lpc,

	gboolean other_changed = FALSE;
	node_t *node = other->action->node;
	+ resource_t *other_rsc = other->action->rsc;
	+ enum rsc_role_e other_role = RSC_ROLE_UNKNOWN;

	+ if(other_rsc) {
	+ other_role = other_rsc->fns->state(other_rsc, TRUE);
	+ }
	+
	do_crm_log(log_level, " Checking action %s: %s %s %s (flags=0x%.6x)",
	other->action->uuid,
	other->action->optional?"optional":"required",
	other->action->runnable?"runnable":"unrunnable",
	other->action->pseudo?"pseudo":other->action->task,
	other->type);

	local_type = other->type;

	- if(local_type & pe_order_demote
	+ if((local_type & pe_order_demote)
	&& other->action->pseudo == FALSE
	- && other->action->rsc->role > RSC_ROLE_SLAVE
	+ && other_role > RSC_ROLE_SLAVE
	&& node != NULL
	&& node->details->online) {
	local_type \|= pe_order_runnable_left;
	do_crm_log(log_level,"Upgrading restart constraint to runnable_left");
	}
	-
	- if(local_type & pe_order_restart
	- && other->action->pseudo == FALSE
	- && node != NULL
	- && node->details->online) {
	-
	- local_type \|= pe_order_implies_left;
	- do_crm_log(log_level,"Upgrading restart constraint to implies_left");

	+ if((local_type & pe_order_shutdown)
	+ && other->action->optional == FALSE
	+ && other_rsc->shutdown) {
	+ action->optional = FALSE;
	+ changed = TRUE;
	+ do_crm_log(log_level-1,
	+ " * Marking action %s manditory because of %s (complex)",
	+ action->uuid, other->action->uuid);
	+ }
	+
	+ if((local_type & pe_order_restart)
	+ && other_role > RSC_ROLE_STOPPED) {
	+
	+ if(other_rsc->variant == pe_native) {
	+ local_type \|= pe_order_implies_left;
	+ do_crm_log(log_level,"Upgrading restart constraint to implies_left");
	+ }
	+
	if(other->action->optional
	&& other->action->runnable
	&& action->runnable == FALSE) {
	do_crm_log(log_level-1,
	" * Marking action %s manditory because %s is unrunnable",
	other->action->uuid, action->uuid);
	other->action->optional = FALSE;
	+ other_rsc->shutdown = TRUE;
	other_changed = TRUE;
	}
	}

	if((local_type & pe_order_runnable_left)
	&& other->action->runnable == FALSE) {
	if(other->action->implied_by_stonith) {
	do_crm_log(log_level, "Ignoring un-runnable - implied_by_stonith");

	} else if(action->runnable == FALSE) {
	do_crm_log(log_level+1, "Already un-runnable");

	} else {
	action->runnable = FALSE;
	do_crm_log(log_level-1,
	" * Marking action %s un-runnable because of %s",
	action->uuid, other->action->uuid);
	changed = TRUE;
	}
	}

	if((local_type & pe_order_runnable_right)
	&& action->runnable == FALSE) {
	if(action->pseudo) {
	do_crm_log(log_level, "Ignoring un-runnable - pseudo");

	} else if(other->action->runnable == FALSE) {
	do_crm_log(log_level+1, "Already un-runnable");

	} else {
	other->action->runnable = FALSE;
	do_crm_log(log_level-1,
	" * Marking action %s un-runnable because of %s",
	other->action->uuid, action->uuid);
	other_changed = TRUE;
	}
	}

	if(local_type & pe_order_implies_left) {
	if(other->action->optional == FALSE) {
	/* nothing to do */
	do_crm_log(log_level+1, " Ignoring implies left - redundant");

	} else if(safe_str_eq(other->action->task, CRMD_ACTION_STOP)
	- && other->action->rsc->fns->state(
	- other->action->rsc, TRUE) == RSC_ROLE_STOPPED) {
	+ && other_role == RSC_ROLE_STOPPED) {
	do_crm_log(log_level-1, " Ignoring implies left - %s already stopped",
	- other->action->rsc->id);
	+ other_rsc->id);

	} else if((local_type & pe_order_demote)
	- && other->action->rsc->role < RSC_ROLE_MASTER) {
	+ && other_rsc->role < RSC_ROLE_MASTER) {
	do_crm_log(log_level-1, " Ignoring implies left - %s already demoted",
	- other->action->rsc->id);
	+ other_rsc->id);

	} else if(action->optional == FALSE) {
	other->action->optional = FALSE;
	do_crm_log(log_level-1,
	" * (implies left) Marking action %s mandatory because of %s",
	other->action->uuid, action->uuid);
	other_changed = TRUE;

	} else {
	do_crm_log(log_level, " Ignoring implies left");
	}
	}

	if(local_type & pe_order_implies_right) {
	if(action->optional == FALSE) {
	/* nothing to do */
	do_crm_log(log_level+1, " Ignoring implies right - redundant");
	} else if(other->action->optional == FALSE) {
	action->optional = FALSE;
	do_crm_log(log_level-1,
	" * (implies right) Marking action %s mandatory because of %s",
	action->uuid, other->action->uuid);
	changed = TRUE;

	} else {
	do_crm_log(log_level, " Ignoring implies right");
	}
	}

	if(other_changed) {
	do_crm_log(log_level, "%s changed, processing after list", other->action->uuid);
	update_action(other->action);
	slist_iter(
	before_other, action_wrapper_t, other->action->actions_after, lpc2,
	do_crm_log(log_level, "%s changed, processing %s", other->action->uuid, before_other->action->uuid);
	update_action(before_other->action);
	);
	}

	);

	if(changed) {
	do_crm_log(log_level, "%s changed, processing after list", action->uuid);
	slist_iter(
	other, action_wrapper_t, action->actions_after, lpc,
	do_crm_log(log_level, "%s changed, processing %s", action->uuid, other->action->uuid);
	update_action(other->action);
	);
	}

	return FALSE;
	}


	gboolean
	shutdown_constraints(
	node_t node, action_t shutdown_op, pe_working_set_t *data_set)
	{
	/* add the stop to the before lists so it counts as a pre-req
	* for the shutdown
	*/
	slist_iter(
	rsc, resource_t, node->details->running_rsc, lpc,

	if(rsc->is_managed == FALSE) {
	continue;
	}

	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL, crm_strdup(CRM_OP_SHUTDOWN), shutdown_op,
	pe_order_implies_left, data_set);

	);

	return TRUE;
	}

	gboolean
	stonith_constraints(
	node_t node, action_t stonith_op, pe_working_set_t *data_set)
	{
	CRM_CHECK(stonith_op != NULL, return FALSE);

	/*
	* Make sure the stonith OP occurs before we start any shared resources
	*/
	if(stonith_op != NULL) {
	slist_iter(
	rsc, resource_t, data_set->resources, lpc,
	rsc->cmds->stonith_ordering(rsc, stonith_op, data_set);
	);
	}

	/* add the stonith OP as a stop pre-req and the mark the stop
	* as a pseudo op - since its now redundant
	*/

	return TRUE;
	}

	static void dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, crm_strdup(key), crm_strdup(value));
	}

	crm_data_t *
	action2xml(action_t *action, gboolean as_input)
	{
	gboolean needs_node_info = TRUE;
	crm_data_t * action_xml = NULL;
	crm_data_t * args_xml = NULL;
	char *action_id_s = NULL;

	if(action == NULL) {
	return NULL;
	}

	crm_debug_4("Dumping action %d as XML", action->id);
	if(safe_str_eq(action->task, CRM_OP_FENCE)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
	/* needs_node_info = FALSE; */

	} else if(safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if(safe_str_eq(action->task, CRM_OP_LRM_REFRESH)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	/* } else if(safe_str_eq(action->task, CRMD_ACTION_PROBED)) { */
	/* action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */

	} else if(action->pseudo) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = FALSE;

	} else {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
	}

	action_id_s = crm_itoa(action->id);
	crm_xml_add(action_xml, XML_ATTR_ID, action_id_s);
	crm_free(action_id_s);

	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
	if(action->rsc != NULL && action->rsc->clone_name != NULL) {
	char *clone_key = NULL;
	const char *interval_s = g_hash_table_lookup(action->meta, "interval");
	int interval = crm_parse_int(interval_s, "0");

	if(safe_str_eq(action->task, CRMD_ACTION_NOTIFY)) {
	const char *n_type = g_hash_table_lookup(
	action->extra, crm_meta_name("notify_type"));
	const char *n_task = g_hash_table_lookup(
	action->extra, crm_meta_name("notify_operation"));
	CRM_CHECK(n_type != NULL, ;);
	CRM_CHECK(n_task != NULL, ;);
	clone_key = generate_notify_key(action->rsc->clone_name, n_type, n_task);

	} else {
	clone_key = generate_op_key(action->rsc->clone_name, action->task, interval);
	}

	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_"XML_LRM_ATTR_TASK_KEY, action->uuid);
	crm_free(clone_key);

	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if(needs_node_info && action->node != NULL) {
	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET,
	action->node->details->uname);

	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET_UUID,
	action->node->details->id);
	}

	if(action->failure_is_fatal == FALSE) {
	add_hash_param(action->meta,
	XML_ATTR_TE_ALLOWFAIL, XML_BOOLEAN_TRUE);
	}

	if(as_input) {
	return action_xml;
	}

	if(action->notify_keys != NULL) {
	g_hash_table_foreach(
	action->notify_keys, dup_attr, action->meta);
	}
	if(action->rsc != NULL && action->pseudo == FALSE) {
	int lpc = 0;

	crm_data_t *rsc_xml = create_xml_node(
	action_xml, crm_element_name(action->rsc->xml));

	const char *attr_list[] = {
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER,
	XML_ATTR_TYPE
	};

	if(action->rsc->clone_name != NULL) {
	crm_debug("Using clone name %s for %s", action->rsc->clone_name, action->rsc->id);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);

	} else {
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->long_name);
	}

	for(lpc = 0; lpc < DIMOF(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	args_xml = create_xml_node(action_xml, XML_TAG_ATTRS);
	crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	g_hash_table_foreach(action->extra, hash2field, args_xml);
	if(action->rsc != NULL && safe_str_neq(action->task, CRMD_ACTION_STOP)) {
	g_hash_table_foreach(action->rsc->parameters, hash2field, args_xml);
	}

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if(action->rsc != NULL) {
	int lpc = 0;
	const char *key = NULL;
	const char *value = NULL;
	const char *meta_list[] = {
	XML_RSC_ATTR_UNIQUE,
	XML_RSC_ATTR_INCARNATION,
	XML_RSC_ATTR_INCARNATION_MAX,
	XML_RSC_ATTR_INCARNATION_NODEMAX,
	XML_RSC_ATTR_MASTER_MAX,
	XML_RSC_ATTR_MASTER_NODEMAX,
	};

	for(lpc = 0; lpc < DIMOF(meta_list); lpc++) {
	key = meta_list[lpc];
	value = g_hash_table_lookup(action->rsc->meta, key);
	if(value != NULL) {
	char *crm_name = crm_concat(CRM_META, key, '_');
	crm_xml_add(args_xml, crm_name, value);
	crm_free(crm_name);
	}
	}
	}

	crm_log_xml_debug_4(action_xml, "dumped action");

	return action_xml;
	}

	static gboolean
	should_dump_action(action_t *action)
	{
	const char * interval = NULL;

	CRM_CHECK(action != NULL, return FALSE);

	interval = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
	if(action->optional) {
	crm_debug_5("action %d (%s) was optional",
	action->id, action->uuid);
	return FALSE;

	} else if(action->runnable == FALSE) {
	crm_debug_5("action %d (%s) was not runnable",
	action->id, action->uuid);
	return FALSE;

	} else if(action->dumped) {
	crm_debug_5("action %d (%s) was already dumped",
	action->id, action->uuid);
	return FALSE;

	} else if(action->rsc != NULL
	&& action->rsc->is_managed == FALSE) {

	/* make sure probes go through */
	if(safe_str_neq(action->task, CRMD_ACTION_STATUS)) {
	pe_warn("action %d (%s) was for an unmanaged resource (%s)",
	action->id, action->uuid, action->rsc->id);
	return FALSE;
	}

	if(interval != NULL && safe_str_neq(interval, "0")) {
	pe_warn("action %d (%s) was for an unmanaged resource (%s)",
	action->id, action->uuid, action->rsc->id);
	return FALSE;
	}
	}

	if(action->pseudo
	\|\| safe_str_eq(action->task, CRM_OP_FENCE)
	\|\| safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
	/* skip the next checks */
	return TRUE;
	}

	if(action->node == NULL) {
	pe_err("action %d (%s) was not allocated",
	action->id, action->uuid);
	log_action(LOG_DEBUG, "Unallocated action", action, FALSE);
	return FALSE;

	} else if(action->node->details->online == FALSE) {
	pe_err("action %d was (%s) scheduled for offline node",
	action->id, action->uuid);
	log_action(LOG_DEBUG, "Action for offline node", action, FALSE);
	return FALSE;
	#if 0
	/* but this would also affect resources that can be safely
	* migrated before a fencing op
	*/
	} else if(action->node->details->unclean == FALSE) {
	pe_err("action %d was (%s) scheduled for unclean node",
	action->id, action->uuid);
	log_action(LOG_DEBUG, "Action for unclean node", action, FALSE);
	return FALSE;
	#endif
	}
	return TRUE;
	}

	/* lowest to highest */
	static gint sort_action_id(gconstpointer a, gconstpointer b)
	{
	const action_wrapper_t action_wrapper2 = (const action_wrapper_t)a;
	const action_wrapper_t action_wrapper1 = (const action_wrapper_t)b;

	if(a == NULL) { return 1; }
	if(b == NULL) { return -1; }

	if(action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}

	if(action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	return 0;
	}

	static gboolean
	should_dump_input(int last_action, action_t action, action_wrapper_t wrapper)
	{
	wrapper->state = pe_link_not_dumped;
	if(last_action == wrapper->action->id) {
	crm_debug_2("Input (%d) %s duplicated",
	wrapper->action->id,
	wrapper->action->uuid);
	wrapper->state = pe_link_dup;
	return FALSE;

	} else if(wrapper->type == pe_order_none) {
	crm_debug_2("Input (%d) %s suppressed",
	wrapper->action->id,
	wrapper->action->uuid);
	return FALSE;

	} else if(wrapper->action->optional == TRUE) {
	crm_debug_2("Input (%d) %s optional",
	wrapper->action->id,
	wrapper->action->uuid);
	return FALSE;

	} else if(wrapper->action->runnable == FALSE
	&& wrapper->action->pseudo == FALSE
	&& wrapper->type == pe_order_optional) {
	crm_debug("Input (%d) %s optional (ordering)",
	wrapper->action->id,
	wrapper->action->uuid);
	return FALSE;

	} else if(action->pseudo
	&& (wrapper->type & pe_order_stonith_stop)) {
	crm_debug("Input (%d) %s suppressed",
	wrapper->action->id,
	wrapper->action->uuid);
	return FALSE;
	}
	crm_debug_3("Input (%d) %s n=%p p=%d r=%d f=0x%.6x dumped for %s",
	wrapper->action->id,
	wrapper->action->uuid,
	wrapper->action->node,
	wrapper->action->pseudo,
	wrapper->action->runnable,
	wrapper->type,
	action->uuid);
	return TRUE;
	}

	void
	graph_element_from_action(action_t action, pe_working_set_t data_set)
	{
	int last_action = -1;
	int synapse_priority = 0;
	crm_data_t * syn = NULL;
	crm_data_t * set = NULL;
	crm_data_t * in = NULL;
	crm_data_t * input = NULL;
	crm_data_t * xml_action = NULL;

	if(should_dump_action(action) == FALSE) {
	return;
	}

	action->dumped = TRUE;

	syn = create_xml_node(data_set->graph, "synapse");
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
	data_set->num_synapse++;

	if(action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if(action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if(synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}

	xml_action = action2xml(action, FALSE);
	add_node_nocopy(set, crm_element_name(xml_action), xml_action);

	action->actions_before = g_list_sort(
	action->actions_before, sort_action_id);

	slist_iter(wrapper,action_wrapper_t,action->actions_before,lpc,

	if(should_dump_input(last_action, action, wrapper) == FALSE) {
	continue;
	}

	wrapper->state = pe_link_dumped;
	CRM_CHECK(last_action < wrapper->action->id, ;);
	last_action = wrapper->action->id;
	input = create_xml_node(in, "trigger");

	xml_action = action2xml(wrapper->action, TRUE);
	add_node_nocopy(input, crm_element_name(xml_action), xml_action);
	);
	}

	diff --git a/crm/pengine/group.c b/crm/pengine/group.c
	index 44892e8269..83d1be8bdf 100644
	--- a/crm/pengine/group.c
	+++ b/crm/pengine/group.c
	@@ -1,399 +1,400 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <lha_internal.h>

	#include <pengine.h>
	#include <lib/crm/pengine/utils.h>
	#include <crm/msg_xml.h>
	#include <clplumbing/cl_misc.h>
	#include <allocate.h>
	#include <utils.h>

	#define VARIANT_GROUP 1
	#include <lib/crm/pengine/variant.h>

	node_t *
	group_color(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *node = NULL;
	node_t *group_node = NULL;
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	if(rsc->provisional == FALSE) {
	return rsc->allocated_to;
	}
	crm_debug_2("Processing %s", rsc->id);
	if(rsc->is_allocating) {
	crm_debug("Dependancy loop detected involving %s", rsc->id);
	return NULL;
	}

	if(group_data->first_child == NULL) {
	/* nothign to allocate */
	rsc->provisional = FALSE;
	return NULL;
	}

	rsc->is_allocating = TRUE;
	rsc->role = group_data->first_child->role;

	group_data->first_child->rsc_cons = g_list_concat(
	group_data->first_child->rsc_cons, rsc->rsc_cons);
	rsc->rsc_cons = NULL;

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	node = child_rsc->cmds->color(child_rsc, data_set);
	if(group_node == NULL) {
	group_node = node;
	}
	);

	rsc->next_role = group_data->first_child->next_role;
	rsc->is_allocating = FALSE;
	rsc->provisional = FALSE;

	if(group_data->colocated) {
	return group_node;
	}
	return NULL;
	}

	void group_update_pseudo_status(resource_t parent, resource_t child);

	void group_create_actions(resource_t rsc, pe_working_set_t data_set)
	{
	action_t *op = NULL;
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	crm_debug_2("Creating actions for %s", rsc->id);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->create_actions(child_rsc, data_set);
	group_update_pseudo_status(rsc, child_rsc);
	);

	op = start_action(rsc, NULL, !group_data->child_starting);
	op->pseudo = TRUE;
	op->runnable = TRUE;

	op = custom_action(rsc, started_key(rsc),
	CRMD_ACTION_STARTED, NULL,
	!group_data->child_starting, TRUE, data_set);
	op->pseudo = TRUE;
	op->runnable = TRUE;

	op = stop_action(rsc, NULL, !group_data->child_stopping);
	op->pseudo = TRUE;
	op->runnable = TRUE;

	op = custom_action(rsc, stopped_key(rsc),
	CRMD_ACTION_STOPPED, NULL,
	!group_data->child_stopping, TRUE, data_set);
	op->pseudo = TRUE;
	op->runnable = TRUE;

	rsc->actions = rsc->actions;
	/* rsc->actions = NULL; */
	}

	void
	group_update_pseudo_status(resource_t parent, resource_t child)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, parent);

	if(group_data->child_stopping && group_data->child_starting) {
	return;
	}
	slist_iter(
	action, action_t, child->actions, lpc,

	if(action->optional) {
	continue;
	}
	if(safe_str_eq(CRMD_ACTION_STOP, action->task) && action->runnable) {
	group_data->child_stopping = TRUE;
	crm_debug_3("Based on %s the group is stopping", action->uuid);

	} else if(safe_str_eq(CRMD_ACTION_START, action->task) && action->runnable) {
	group_data->child_starting = TRUE;
	crm_debug_3("Based on %s the group is starting", action->uuid);
	}

	);
	}

	void group_internal_constraints(resource_t rsc, pe_working_set_t data_set)
	{
	resource_t *last_rsc = NULL;
	-
	+ int stopstop = pe_order_shutdown;
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	native_internal_constraints(rsc, data_set);
	+
	+ if(group_data->ordered == FALSE) {
	+ stopstop \|= pe_order_implies_right;
	+ }

	custom_action_order(
	rsc, stopped_key(rsc), NULL,
	rsc, start_key(rsc), NULL,
	pe_order_optional, data_set);

	custom_action_order(
	rsc, stop_key(rsc), NULL,
	rsc, stopped_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	custom_action_order(
	rsc, start_key(rsc), NULL,
	rsc, started_key(rsc), NULL,
	pe_order_runnable_left, data_set);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	if(group_data->colocated && last_rsc != NULL) {
	rsc_colocation_new(
	"group:internal_colocation", NULL, INFINITY,
	child_rsc, last_rsc, NULL, NULL, data_set);
	}

	- custom_action_order(rsc, stop_key(rsc), NULL,
	- child_rsc, stop_key(child_rsc), NULL,
	- pe_order_optional, data_set);
	+ order_stop_stop(rsc, child_rsc, stopstop);

	custom_action_order(child_rsc, stop_key(child_rsc), NULL,
	rsc, stopped_key(rsc), NULL,
	pe_order_optional, data_set);

	custom_action_order(child_rsc, start_key(child_rsc), NULL,
	rsc, started_key(rsc), NULL,
	- pe_order_optional, data_set);
	+ pe_order_runnable_left, data_set);

	if(group_data->ordered == FALSE) {
	order_start_start(rsc, child_rsc, pe_order_implies_right\|pe_order_runnable_left);
	- order_stop_stop(rsc, child_rsc, pe_order_implies_right);

	} else if(last_rsc != NULL) {
	order_start_start(last_rsc, child_rsc, pe_order_implies_right\|pe_order_runnable_left);
	order_stop_stop(child_rsc, last_rsc, pe_order_implies_left);

	child_rsc->restart_type = pe_restart_restart;

	} else {
	/* If anyone in the group is starting, then
	* pe_order_implies_right will cause _everyone_ in the group
	* to be sent a start action
	* But this is safe since starting something that is already
	* started is required to be "safe"
	*/
	order_start_start(rsc, child_rsc,
	pe_order_implies_right\|pe_order_implies_left\|pe_order_runnable_right\|pe_order_runnable_left);
	}

	last_rsc = child_rsc;
	);

	if(group_data->ordered && last_rsc != NULL) {
	order_stop_stop(rsc, last_rsc, pe_order_implies_right);
	}
	}


	void group_rsc_colocation_lh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	group_variant_data_t *group_data = NULL;

	if(rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if(rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	crm_debug_4("Processing constraints from %s", rsc_lh->id);

	get_group_variant_data(group_data, rsc_lh);

	if(group_data->colocated) {
	group_data->first_child->cmds->rsc_colocation_lh(
	group_data->first_child, rsc_rh, constraint);
	return;

	} else if(constraint->score >= INFINITY) {
	crm_config_err("%s: Cannot perform manditory colocation"
	" between non-colocated group and %s",
	rsc_lh->id, rsc_rh->id);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc_lh->children, lpc,
	child_rsc->cmds->rsc_colocation_lh(
	child_rsc, rsc_rh, constraint);
	);
	}

	void group_rsc_colocation_rh(
	resource_t rsc_lh, resource_t rsc_rh, rsc_colocation_t *constraint)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc_rh);
	CRM_CHECK(rsc_lh->variant == pe_native, return);

	crm_debug_3("Processing RH of constraint %s", constraint->id);
	print_resource(LOG_DEBUG_3, "LHS", rsc_lh, TRUE);

	if(rsc_rh->provisional) {
	return;

	} else if(group_data->colocated && group_data->first_child) {
	group_data->first_child->cmds->rsc_colocation_rh(
	rsc_lh, group_data->first_child, constraint);
	return;

	} else if(constraint->score >= INFINITY) {
	crm_config_err("%s: Cannot perform manditory colocation with"
	" non-colocated group: %s", rsc_lh->id, rsc_rh->id);
	return;
	}

	slist_iter(
	child_rsc, resource_t, rsc_rh->children, lpc,
	child_rsc->cmds->rsc_colocation_rh(
	rsc_lh, child_rsc, constraint);
	);
	}

	void group_rsc_order_lh(resource_t rsc, order_constraint_t order, pe_working_set_t *data_set)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	crm_debug_2("%s->%s", order->lh_action_task, order->rh_action_task);

	if(order->rh_rsc != NULL
	&& (rsc == order->rh_rsc \|\| rsc == order->rh_rsc->parent)) {
	native_rsc_order_lh(rsc, order, data_set);
	return;
	}

	if(order->type != pe_order_optional) {
	native_rsc_order_lh(rsc, order, data_set);
	}

	if(order->type & pe_order_implies_left) {
	native_rsc_order_lh(group_data->first_child, order, data_set);
	}

	convert_non_atomic_task(rsc, order);
	native_rsc_order_lh(rsc, order, data_set);
	}

	void group_rsc_order_rh(
	action_t lh_action, resource_t rsc, order_constraint_t *order)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	crm_debug_2("%s->%s", lh_action->uuid, order->rh_action_task);

	if(rsc == NULL) {
	return;
	}

	native_rsc_order_rh(lh_action, rsc, order);
	}

	void group_rsc_location(resource_t rsc, rsc_to_node_t constraint)
	{
	gboolean reset_scores = TRUE;
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	crm_debug("Processing rsc_location %s for %s",
	constraint->id, rsc->id);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,
	child_rsc->cmds->rsc_location(child_rsc, constraint);
	if(group_data->colocated && reset_scores) {
	reset_scores = FALSE;
	slist_iter(node, node_t, constraint->node_list_rh, lpc2,
	node->weight = 0;
	);
	}
	);
	}

	void group_expand(resource_t rsc, pe_working_set_t data_set)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	crm_debug_3("Processing actions from %s", rsc->id);

	CRM_CHECK(rsc != NULL, return);
	native_expand(rsc, data_set);

	slist_iter(
	child_rsc, resource_t, rsc->children, lpc,

	child_rsc->cmds->expand(child_rsc, data_set);
	);

	}

	GListPtr
	group_merge_weights(
	resource_t rsc, const char rhs, GListPtr nodes, int factor, gboolean allow_rollback)
	{
	group_variant_data_t *group_data = NULL;
	get_group_variant_data(group_data, rsc);

	if(rsc->is_merging) {
	crm_debug("Breaking dependancy loop with %s at %s", rsc->id, rhs);
	return nodes;

	} else if(rsc->provisional == FALSE \|\| can_run_any(nodes) == FALSE) {
	return nodes;
	}

	rsc->is_merging = TRUE;
	nodes = group_data->first_child->cmds->merge_weights(
	group_data->first_child, rhs, nodes, factor, allow_rollback);

	slist_iter(
	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,

	nodes = native_merge_weights(
	constraint->rsc_lh, rsc->id, nodes,
	constraint->score/INFINITY, allow_rollback);
	);

	rsc->is_merging = FALSE;
	return nodes;
	}
	diff --git a/crm/pengine/pengine.h b/crm/pengine/pengine.h
	index 2bbfb8fa49..c2d5c3c110 100644
	--- a/crm/pengine/pengine.h
	+++ b/crm/pengine/pengine.h
	@@ -1,174 +1,176 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef PENGINE__H
	#define PENGINE__H

	#include <clplumbing/ipc.h>

	typedef struct rsc_to_node_s rsc_to_node_t;
	typedef struct rsc_colocation_s rsc_colocation_t;
	typedef struct lrm_agent_s lrm_agent_t;
	typedef struct order_constraint_s order_constraint_t;

	#include <glib.h>
	#include <crm/crm.h>
	#include <crm/common/msg.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/common.h>
	#include <crm/pengine/status.h>

	#include <crm/pengine/complex.h>

	enum pe_stop_fail {
	pesf_block,
	pesf_stonith,
	pesf_ignore
	};

	enum pe_ordering {
	pe_order_none = 0x0, /* deleted */

	pe_order_implies_left = 0x01, /* was: _mandatory */
	pe_order_implies_right = 0x02, /* was: _recover */

	pe_order_runnable_left = 0x10, /* needs the LHS side to be runnable */
	pe_order_runnable_right = 0x20, /* needs the RHS side to be runnable */

	pe_order_optional = 0x100, /* pure ordering, nothing implied */
	pe_order_stonith_stop = 0x200, /* only applies if the action is non-pseudo */
	pe_order_restart = 0x400, /* stop-start constraint */
	pe_order_demote = 0x800, /* stop-start constraint */

	- pe_order_test = 0x1000 /* test marker */
	+ pe_order_shutdown = 0x1000, /* combines with pe_order_restart to make a complex resource shut down */
	+
	+ pe_order_test = 0x10000 /* test marker */
	};

	struct rsc_colocation_s {
	const char *id;
	const char *node_attribute;
	resource_t *rsc_lh;
	resource_t *rsc_rh;

	int role_lh;
	int role_rh;

	int score;
	};

	struct rsc_to_node_s {
	const char *id;
	resource_t *rsc_lh;

	enum rsc_role_e role_filter;
	GListPtr node_list_rh; /* node_t* */
	};


	struct order_constraint_s
	{
	int id;
	enum pe_ordering type;

	void *lh_opaque;
	resource_t *lh_rsc;
	action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	resource_t *rh_rsc;
	action_t *rh_action;
	char *rh_action_task;

	/* (soon to be) variant specific */
	/* int lh_rsc_incarnation; */
	/* int rh_rsc_incarnation; */
	};

	enum pe_link_state {
	pe_link_not_dumped,
	pe_link_dumped,
	pe_link_dup,
	};

	typedef struct action_wrapper_s action_wrapper_t;
	struct action_wrapper_s
	{
	enum pe_ordering type;
	enum pe_link_state state;
	action_t *action;
	};

	extern gboolean stage0(pe_working_set_t *data_set);
	extern gboolean stage1(pe_working_set_t *data_set);
	extern gboolean stage2(pe_working_set_t *data_set);
	extern gboolean stage3(pe_working_set_t *data_set);
	extern gboolean stage4(pe_working_set_t *data_set);
	extern gboolean stage5(pe_working_set_t *data_set);
	extern gboolean stage6(pe_working_set_t *data_set);
	extern gboolean stage7(pe_working_set_t *data_set);
	extern gboolean stage8(pe_working_set_t *data_set);

	extern gboolean summary(GListPtr resources);

	extern gboolean pe_msg_dispatch(IPC_Channel sender, void user_data);

	extern gboolean process_pe_message(
	HA_Message msg, crm_data_t xml_data, IPC_Channel *sender);

	extern gboolean unpack_constraints(
	crm_data_t xml_constraints, pe_working_set_t data_set);

	extern gboolean update_action_states(GListPtr actions);

	extern gboolean shutdown_constraints(
	node_t node, action_t shutdown_op, pe_working_set_t *data_set);

	extern gboolean stonith_constraints(
	node_t node, action_t stonith_op, pe_working_set_t *data_set);

	extern int custom_action_order(
	resource_t lh_rsc, char lh_task, action_t *lh_action,
	resource_t rh_rsc, char rh_task, action_t *rh_action,
	enum pe_ordering type, pe_working_set_t *data_set);

	#define order_start_start(rsc1,rsc2, type) \
	custom_action_order(rsc1, start_key(rsc1), NULL, \
	rsc2, start_key(rsc2) ,NULL, \
	type, data_set)
	#define order_stop_stop(rsc1, rsc2, type) \
	custom_action_order(rsc1, stop_key(rsc1), NULL, \
	rsc2, stop_key(rsc2) ,NULL, \
	type, data_set)

	#define order_stop_start(rsc1, rsc2, type) \
	custom_action_order(rsc1, stop_key(rsc1), NULL, \
	rsc2, start_key(rsc2) ,NULL, \
	type, data_set)

	#define order_start_stop(rsc1, rsc2, type) \
	custom_action_order(rsc1, start_key(rsc1), NULL, \
	rsc2, stop_key(rsc2) ,NULL, \
	type, data_set)

	extern void graph_element_from_action(
	action_t action, pe_working_set_t data_set);

	extern const char* transition_idle_timeout;

	#endif

	diff --git a/crm/pengine/regression.sh b/crm/pengine/regression.sh
	index d866b7d5d4..704331a0e3 100755
	--- a/crm/pengine/regression.sh
	+++ b/crm/pengine/regression.sh
	@@ -1,268 +1,269 @@
	#!/bin/bash

	# Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public
	# License as published by the Free Software Foundation; either
	# version 2.1 of the License, or (at your option) any later version.
	#
	# This software is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# General Public License for more details.
	#
	# You should have received a copy of the GNU General Public
	# License along with this library; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	#
	if [ -x /usr/bin/valgrind ]; then
	export G_SLICE=always-malloc
	VALGRIND_CMD="valgrind -q --show-reachable=yes --leak-check=full --trace-children=no --time-stamp=yes --num-callers=20 --suppressions=./ptest.supp"
	fi

	. regression.core.sh
	create_mode="true"
	echo Generating test outputs for these tests...

	# do_test

	echo Done.
	echo ""

	echo Performing the following tests...
	create_mode="false"

	echo ""
	do_test simple1 "Offline "
	do_test simple2 "Start "
	do_test simple3 "Start 2 "
	do_test simple4 "Start Failed"
	do_test simple6 "Stop Start "
	do_test simple7 "Shutdown "
	#do_test simple8 "Stonith "
	#do_test simple9 "Lower version"
	#do_test simple10 "Higher version"
	do_test simple11 "Priority (ne)"
	do_test simple12 "Priority (eq)"
	do_test simple8 "Stickiness"

	echo ""
	do_test params-0 "Params: No change"
	do_test params-1 "Params: Changed"
	do_test params-2 "Params: Resource definition"
	do_test params-4 "Params: Reload"
	do_test novell-251689 "Resource definition change + target_role=stopped"

	echo ""
	do_test orphan-0 "Orphan ignore"
	do_test orphan-1 "Orphan stop"

	echo ""
	do_test target-0 "Target Role : baseline"
	do_test target-1 "Target Role : test"

	echo ""
	do_test date-1 "Dates" -d "2005-020"
	do_test date-2 "Date Spec - Pass" -d "2005-020T12:30"
	do_test date-3 "Date Spec - Fail" -d "2005-020T11:30"
	do_test probe-0 "Probe (anon clone)"
	do_test probe-1 "Pending Probe"
	do_test standby "Standby"
	do_test comments "Comments"

	echo ""
	do_test rsc_dep1 "Must not "
	do_test rsc_dep3 "Must "
	do_test rsc_dep5 "Must not 3 "
	do_test rsc_dep7 "Must 3 "
	do_test rsc_dep10 "Must (but cant)"
	do_test rsc_dep2 "Must (running) "
	do_test rsc_dep8 "Must (running : alt) "
	do_test rsc_dep4 "Must (running + move)"

	echo ""
	do_test order1 "Order start 1 "
	do_test order2 "Order start 2 "
	do_test order3 "Order stop "
	do_test order4 "Order (multiple) "
	do_test order5 "Order (move) "
	do_test order6 "Order (move w/ restart) "
	do_test order7 "Order (manditory) "
	do_test order-optional "Order (score=0) "
	do_test order-required "Order (score=INFINITY) "

	echo ""
	do_test coloc-loop "Colocation - loop"
	do_test coloc-many-one "Colocation - many-to-one"
	do_test coloc-list "Colocation - many-to-one with list"
	do_test coloc-group "Colocation - groups"

	#echo ""
	#do_test agent1 "version: lt (empty)"
	#do_test agent2 "version: eq "
	#do_test agent3 "version: gt "

	echo ""
	do_test attrs1 "string: eq (and) "
	do_test attrs2 "string: lt / gt (and)"
	do_test attrs3 "string: ne (or) "
	do_test attrs4 "string: exists "
	do_test attrs5 "string: not_exists "
	do_test attrs6 "is_dc: true "
	do_test attrs7 "is_dc: false "
	do_test attrs8 "score_attribute "

	echo ""
	do_test mon-rsc-1 "Schedule Monitor - start"
	do_test mon-rsc-2 "Schedule Monitor - move "
	do_test mon-rsc-3 "Schedule Monitor - pending start "
	do_test mon-rsc-4 "Schedule Monitor - move/pending start"

	echo ""
	do_test rec-rsc-0 "Resource Recover - no start "
	do_test rec-rsc-1 "Resource Recover - start "
	do_test rec-rsc-2 "Resource Recover - monitor "
	do_test rec-rsc-3 "Resource Recover - stop - ignore"
	do_test rec-rsc-4 "Resource Recover - stop - block "
	do_test rec-rsc-5 "Resource Recover - stop - fence "
	do_test rec-rsc-6 "Resource Recover - multiple - restart"
	do_test rec-rsc-7 "Resource Recover - multiple - stop "
	do_test rec-rsc-8 "Resource Recover - multiple - block "
	do_test rec-rsc-9 "Resource Recover - group/group"

	echo ""
	do_test quorum-1 "No quorum - ignore"
	do_test quorum-2 "No quorum - freeze"
	do_test quorum-3 "No quorum - stop "
	do_test quorum-4 "No quorum - start anyway"
	do_test quorum-5 "No quorum - start anyway (group)"
	do_test quorum-6 "No quorum - start anyway (clone)"

	echo ""
	do_test rec-node-1 "Node Recover - Startup - no fence"
	do_test rec-node-2 "Node Recover - Startup - fence "
	do_test rec-node-3 "Node Recover - HA down - no fence"
	do_test rec-node-4 "Node Recover - HA down - fence "
	do_test rec-node-5 "Node Recover - CRM down - no fence"
	do_test rec-node-6 "Node Recover - CRM down - fence "
	do_test rec-node-7 "Node Recover - no quorum - ignore "
	do_test rec-node-8 "Node Recover - no quorum - freeze "
	do_test rec-node-9 "Node Recover - no quorum - stop "
	do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
	do_test rec-node-11 "Node Recover - CRM down w/ group - fence "
	do_test rec-node-12 "Node Recover - nothing active - fence "
	do_test rec-node-13 "Node Recover - failed resource + shutdown - fence "
	do_test rec-node-15 "Node Recover - unknown lrm section"
	do_test rec-node-14 "Serialize all stonith's"

	echo ""
	do_test multi1 "Multiple Active (stop/start)"

	echo ""
	do_test migrate-1 "Migrate (migrate)"
	do_test migrate-2 "Migrate (stable)"
	do_test migrate-3 "Migrate (failed migrate_to)"
	do_test migrate-4 "Migrate (failed migrate_from)"
	do_test novell-252693 "Migration in a stopping stack"
	do_test novell-252693-2 "Migration in a starting stack"
	do_test novell-252693-3 "Non-Migration in a starting and stopping stack"

	#echo ""
	#do_test complex1 "Complex "

	echo ""
	do_test group1 "Group "
	do_test group2 "Group + Native "
	do_test group3 "Group + Group "
	do_test group4 "Group + Native (nothing)"
	do_test group5 "Group + Native (move) "
	do_test group6 "Group + Group (move) "
	do_test group7 "Group colocation"
	do_test group13 "Group colocation (cant run)"
	do_test group8 "Group anti-colocation"
	do_test group9 "Group recovery"
	do_test group10 "Group partial recovery"
	do_test group11 "Group target_role"
	do_test group14 "Group stop (graph terminated)"
	do_test group15 "-ve group colocation"
	do_test bug-1573 "Partial stop of a group with two children"
	+do_test bug-1718 "Mandatory group ordering - Stop group_FUN"

	echo ""
	do_test inc0 "Incarnation start"
	do_test inc1 "Incarnation start order"
	do_test inc2 "Incarnation silent restart, stop, move"
	do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
	do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
	do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
	do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
	do_test inc7 "Clone colocation"
	do_test inc8 "Clone anti-colocation"
	do_test inc9 "Non-unique clone"
	do_test inc10 "Non-unique clone (stop)"
	do_test inc11 "Primitive colocation with clones"
	do_test inc12 "Clone shutdown"
	do_test cloned-group "Make sure only the correct number of cloned groups are started"

	echo ""
	do_test master-0 "Stopped -> Slave"
	do_test master-1 "Stopped -> Promote"
	do_test master-2 "Stopped -> Promote : notify"
	do_test master-3 "Stopped -> Promote : master location"
	do_test master-4 "Started -> Promote : master location"
	do_test master-5 "Promoted -> Promoted"
	do_test master-6 "Promoted -> Promoted (2)"
	do_test master-7 "Promoted -> Fenced"
	do_test master-8 "Promoted -> Fenced -> Moved"
	do_test master-9 "Stopped + Promotable + No quorum"
	do_test master-10 "Stopped -> Promotable : notify with monitor"
	do_test master-11 "Stopped -> Promote : colocation"
	do_test novell-239082 "Demote/Promote ordering"
	do_test novell-239087 "Stable master placement"
	do_test master-12 "Promotion based solely on rsc_location constraints"
	do_test master-13 "Include preferences of colocated resources when placing master"
	do_test master-demote "Ordering when actions depends on demoting a slave resource"
	do_test master-ordering "Prevent resources from starting that need a master"

	echo ""
	do_test managed-0 "Managed (reference)"
	do_test managed-1 "Not managed - down "
	do_test managed-2 "Not managed - up "

	echo ""
	do_test interleave-0 "Interleave (reference)"
	do_test interleave-1 "coloc - not interleaved"
	do_test interleave-2 "coloc - interleaved "
	do_test interleave-3 "coloc - interleaved (2)"

	echo ""
	do_test notify-0 "Notify reference"
	do_test notify-1 "Notify simple"
	do_test notify-2 "Notify simple, confirm"
	do_test notify-3 "Notify move, confirm"
	do_test novell-239079 "Notification priority"
	#do_test notify-2 "Notify - 764"

	echo ""
	do_test 594 "OSDL #594"
	do_test 662 "OSDL #662"
	do_test 696 "OSDL #696"
	do_test 726 "OSDL #726"
	do_test 735 "OSDL #735"
	do_test 764 "OSDL #764"
	do_test 797 "OSDL #797"
	do_test 829 "OSDL #829"
	do_test 994 "OSDL #994"
	do_test 1360 "OSDL #1360 - Clone stickiness"
	do_test 1484 "OSDL #1484 - on_fail=stop"
	do_test 1494 "OSDL #1494 - Clone stability"
	do_test unrunnable-1 "Unrunnable"
	do_test stonith-0 "Stonith loop - 1"
	do_test stonith-1 "Stonith loop - 2"
	do_test bug-1572-1 "Recovery of groups depending on master/slave"
	do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
	do_test bug-1685 "Depends-on-master ordering"

	echo ""

	test_results
	diff --git a/crm/pengine/testcases/bug-1718.dot b/crm/pengine/testcases/bug-1718.dot
	new file mode 100644
	index 0000000000..9e660623f8
	--- /dev/null
	+++ b/crm/pengine/testcases/bug-1718.dot
	@@ -0,0 +1,19 @@
	+digraph "g" {
	+"all_stopped" [ style=bold color="green" fontcolor="orange" ]
	+"group_fUN_start_0" -> "resource_IP3_start_0 ops.ds9" [ style = dashed]
	+"group_fUN_start_0" [ style=dashed color="red" fontcolor="orange" ]
	+"group_fUN_stop_0" -> "group_fUN_start_0" [ style = dashed]
	+"group_fUN_stop_0" -> "resource_IP3_stop_0 ops.ds9" [ style = bold]
	+"group_fUN_stop_0" -> "resource_dummy_stop_0 ops.ds9" [ style = bold]
	+"group_fUN_stop_0" [ style=bold color="green" fontcolor="orange" ]
	+"resource_IP3_start_0 ops.ds9" -> "resource_dummy_start_0 ops.ds9" [ style = dashed]
	+"resource_IP3_start_0 ops.ds9" [ style=dashed color="red" fontcolor="black" ]
	+"resource_IP3_stop_0 ops.ds9" -> "all_stopped" [ style = bold]
	+"resource_IP3_stop_0 ops.ds9" -> "resource_IP3_start_0 ops.ds9" [ style = dashed]
	+"resource_IP3_stop_0 ops.ds9" [ style=bold color="green" fontcolor="black" ]
	+"resource_dummy_start_0 ops.ds9" [ style=dashed color="red" fontcolor="black" ]
	+"resource_dummy_stop_0 ops.ds9" -> "all_stopped" [ style = bold]
	+"resource_dummy_stop_0 ops.ds9" -> "resource_IP3_stop_0 ops.ds9" [ style = bold]
	+"resource_dummy_stop_0 ops.ds9" -> "resource_dummy_start_0 ops.ds9" [ style = dashed]
	+"resource_dummy_stop_0 ops.ds9" [ style=bold color="green" fontcolor="black" ]
	+}
	diff --git a/crm/pengine/testcases/bug-1718.exp b/crm/pengine/testcases/bug-1718.exp
	new file mode 100644
	index 0000000000..151ba878e8
	--- /dev/null
	+++ b/crm/pengine/testcases/bug-1718.exp
	@@ -0,0 +1,55 @@
	+ <transition_graph cluster-delay="60s" transition_id="0">
	+ <synapse id="0">
	+ <action_set>
	+ <pseudo_event id="18" operation="stop" operation_key="group_fUN_stop_0">
	+ <attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs/>
	+ </synapse>
	+ <synapse id="1">
	+ <action_set>
	+ <rsc_op id="12" operation="stop" operation_key="resource_IP3_stop_0" on_node="ops.ds9" on_node_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad">
	+ <primitive id="resource_IP3" long-id="group_fUN:resource_IP3" class="ocf" provider="heartbeat" type="IPaddr"/>
	+ <attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="14" operation="stop" operation_key="resource_dummy_stop_0" on_node="ops.ds9" on_node_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad"/>
	+ </trigger>
	+ <trigger>
	+ <pseudo_event id="18" operation="stop" operation_key="group_fUN_stop_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="2">
	+ <action_set>
	+ <rsc_op id="14" operation="stop" operation_key="resource_dummy_stop_0" on_node="ops.ds9" on_node_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad">
	+ <primitive id="resource_dummy" long-id="group_fUN:resource_dummy" class="ocf" provider="heartbeat" type="Dummy"/>
	+ <attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	+ </rsc_op>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <pseudo_event id="18" operation="stop" operation_key="group_fUN_stop_0"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ <synapse id="3">
	+ <action_set>
	+ <pseudo_event id="5" operation="all_stopped" operation_key="all_stopped">
	+ <attributes crm_feature_set="2.0"/>
	+ </pseudo_event>
	+ </action_set>
	+ <inputs>
	+ <trigger>
	+ <rsc_op id="12" operation="stop" operation_key="resource_IP3_stop_0" on_node="ops.ds9" on_node_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad"/>
	+ </trigger>
	+ <trigger>
	+ <rsc_op id="14" operation="stop" operation_key="resource_dummy_stop_0" on_node="ops.ds9" on_node_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad"/>
	+ </trigger>
	+ </inputs>
	+ </synapse>
	+ </transition_graph>
	+
	diff --git a/crm/pengine/testcases/bug-1718.xml b/crm/pengine/testcases/bug-1718.xml
	new file mode 100644
	index 0000000000..9413d8b505
	--- /dev/null
	+++ b/crm/pengine/testcases/bug-1718.xml
	@@ -0,0 +1,203 @@
	+ <cib generated="true" admin_epoch="0" have_quorum="true" ignore_dtd="false" num_peers="5" cib_feature_revision="1.3" ccm_transition="6" dc_uuid="1e550a7a-16a2-453a-81c3-8d6e04919dad" epoch="98" num_updates="20">
	+ <configuration>
	+ <crm_config>
	+ <cluster_property_set id="cib-bootstrap-options">
	+ <attributes>
	+ <nvpair name="last-lrm-refresh" id="cib-bootstrap-options-last-lrm-refresh" value="1192999094"/>
	+ </attributes>
	+ </cluster_property_set>
	+ </crm_config>
	+ <nodes>
	+ <node id="11e02303-5984-4f6d-be06-37e137335612" uname="defiant.ds9" type="normal">
	+ <instance_attributes id="nodes-11e02303-5984-4f6d-be06-37e137335612">
	+ <attributes>
	+ <nvpair id="standby-11e02303-5984-4f6d-be06-37e137335612" name="standby" value="on"/>
	+ </attributes>
	+ </instance_attributes>
	+ </node>
	+ <node id="d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57" uname="heartbeat.ds9" type="normal"/>
	+ <node id="1e550a7a-16a2-453a-81c3-8d6e04919dad" uname="ops.ds9" type="normal"/>
	+ <node id="ed7370db-6624-4d23-a723-9fafcd031f21" uname="defiant.ds9" type="normal"/>
	+ <node id="de2a634c-e405-42bb-9df2-292e36ad5970" uname="biggame.ds9" type="normal"/>
	+ <node id="674234bf-84a9-4059-8d48-28e273cc280e" uname="warbird.ds9" type="normal"/>
	+ </nodes>
	+ <resources>
	+ <group ordered="true" collocated="true" id="Web_Group">
	+ <primitive class="ocf" type="IPaddr" provider="heartbeat" id="Apache_IP">
	+ <instance_attributes id="Apache_IP_instance_attrs">
	+ <attributes>
	+ <nvpair name="target_role" id="Apache_IP_target_role" value="started"/>
	+ <nvpair id="Apache_IP_ip" name="ip" value="10.0.0.180"/>
	+ <nvpair id="Apache_IP_cidr_netmask" name="cidr_netmask" value="255.255.255.0"/>
	+ <nvpair id="Apache_IP_broadcast" name="broadcast" value="10.0.0.255"/>
	+ </attributes>
	+ </instance_attributes>
	+ </primitive>
	+ <instance_attributes id="Web_Group_instance_attrs">
	+ <attributes>
	+ <nvpair id="Web_Group_target_role" name="target_role" value="started"/>
	+ </attributes>
	+ </instance_attributes>
	+ <primitive class="ocf" type="IPaddr" provider="heartbeat" id="resource_IP2">
	+ <instance_attributes id="resource_IP2_instance_attrs">
	+ <attributes>
	+ <nvpair name="target_role" id="resource_IP2_target_role" value="stopped"/>
	+ <nvpair id="7680eb0a-0f4d-4369-89b8-f371d1224873" name="ip" value="10.0.0.67"/>
	+ <nvpair id="e399715a-2fc4-44fc-a327-0e20b0ace704" name="cidr_netmask" value="255.255.255.0"/>
	+ <nvpair id="c504e392-c889-450e-b035-95b5474233c8" name="broadcast" value="10.0.0.255"/>
	+ </attributes>
	+ </instance_attributes>
	+ </primitive>
	+ <primitive id="resource_dummyweb" class="ocf" type="Dummy" provider="heartbeat">
	+ <instance_attributes id="resource_dummyweb_instance_attrs">
	+ <attributes>
	+ <nvpair id="resource_dummyweb_target_role" name="target_role" value="started"/>
	+ </attributes>
	+ </instance_attributes>
	+ </primitive>
	+ </group>
	+ <group ordered="true" collocated="true" id="group_fUN">
	+ <primitive id="resource_IP3" class="ocf" type="IPaddr" provider="heartbeat">
	+ <instance_attributes id="resource_IP3_instance_attrs">
	+ <attributes>
	+ <nvpair id="resource_IP3_target_role" name="target_role" value="started"/>
	+ <nvpair id="961cbc4e-fd17-4f30-a933-3db77ba69407" name="ip" value="10.0.0.68"/>
	+ <nvpair id="d329e99a-456f-4689-8a5e-f80178349d4f" name="cidr_netmask" value="255.255.255.0"/>
	+ <nvpair id="49f4cf40-6987-4e14-aace-28ab5d776953" name="broadcast" value="10.0.0.255"/>
	+ </attributes>
	+ </instance_attributes>
	+ </primitive>
	+ <primitive class="ocf" type="Dummy" provider="heartbeat" id="resource_dummy">
	+ <instance_attributes id="resource_dummy_instance_attrs">
	+ <attributes>
	+ <nvpair name="target_role" id="resource_dummy_target_role" value="started"/>
	+ </attributes>
	+ </instance_attributes>
	+ </primitive>
	+ </group>
	+ </resources>
	+ <constraints>
	+ <rsc_location id="Web_Place" rsc="Web_Group">
	+ <rule id="prefered_Web_Place" score="100">
	+ <expression attribute="#uname" id="f0c82530-5984-42f8-ac40-8ce32048f2f9" operation="eq" value="defiant.ds9"/>
	+ </rule>
	+ </rsc_location>
	+ <rsc_order id="order_Web_before_fun" from="Web_Group" type="before" to="group_fUN"/>
	+ </constraints>
	+ </configuration>
	+ <status>
	+ <node_state uname="heartbeat.ds9" crmd="online" shutdown="0" in_ccm="true" join="member" ha="active" expected="member" id="d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57" crm-debug-origin="do_update_resource">
	+ <transient_attributes id="d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57">
	+ <instance_attributes id="status-d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57">
	+ <attributes>
	+ <nvpair id="status-d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57-probe_complete" name="probe_complete" value="true"/>
	+ <nvpair id="status-d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57-fail-count-OpenVPN" name="fail-count-OpenVPN" value="1"/>
	+ </attributes>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="d9fea871-5fc2-4c5d-b3bd-c830fc5a5d57">
	+ <lrm_resources>
	+ <lrm_resource id="OpenVPN_IP" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="OpenVPN_IP_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="7:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;7:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="6" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="1f0bbac744caf5cfdfd4c698739d09c5"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_IP2" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="3:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;3:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="49" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="368d7680070c38f8f2f40ddf5a434d45"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache_IP" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_IP_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;5:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="4" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="a3558b7ea398267ef1ea56b21f5d840a"/>
	+ <lrm_rsc_op id="Apache_IP_start_0" operation="start" crm-debug-origin="do_update_resource" transition_key="6:0:5e45f285-3200-4d7f-966a-42c650d75a6c" transition_magic="0:0;6:0:5e45f285-3200-4d7f-966a-42c650d75a6c" call_id="75" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="a3558b7ea398267ef1ea56b21f5d840a"/>
	+ <lrm_rsc_op id="Apache_IP_stop_0" operation="stop" crm-debug-origin="build_active_RAs" transition_key="19:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;19:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="17" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="a3558b7ea398267ef1ea56b21f5d840a"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_IP3" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="3:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;3:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="50" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ <lrm_rsc_op id="resource_IP3_stop_0" operation="stop" crm-debug-origin="do_update_resource" transition_key="31:0:5e45f285-3200-4d7f-966a-42c650d75a6c" transition_magic="0:0;31:0:5e45f285-3200-4d7f-966a-42c650d75a6c" call_id="77" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ <lrm_rsc_op id="resource_IP3_start_0" operation="start" crm-debug-origin="build_active_RAs" transition_key="40:87:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;40:87:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="73" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache" type="apache13" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="3:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:1;3:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="16" crm_feature_set="2.0" rc_code="1" op_status="4" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ <lrm_rsc_op id="Apache_stop_0" operation="stop" crm-debug-origin="build_active_RAs" transition_key="1:23:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;1:23:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="18" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummyweb" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummyweb_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="3:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;3:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="62" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummy" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="3:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;3:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="52" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="resource_dummy_stop_0" operation="stop" crm-debug-origin="do_update_resource" transition_key="33:0:5e45f285-3200-4d7f-966a-42c650d75a6c" transition_magic="0:0;33:0:5e45f285-3200-4d7f-966a-42c650d75a6c" call_id="76" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="resource_dummy_start_0" operation="start" crm-debug-origin="build_active_RAs" transition_key="42:87:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;42:87:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="74" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op_force_restart=" state " op_restart_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state uname="ops.ds9" crmd="online" in_ccm="true" join="member" ha="active" expected="member" shutdown="0" id="1e550a7a-16a2-453a-81c3-8d6e04919dad" crm-debug-origin="do_update_resource">
	+ <transient_attributes id="1e550a7a-16a2-453a-81c3-8d6e04919dad">
	+ <instance_attributes id="status-1e550a7a-16a2-453a-81c3-8d6e04919dad">
	+ <attributes>
	+ <nvpair id="status-1e550a7a-16a2-453a-81c3-8d6e04919dad-probe_complete" name="probe_complete" value="true"/>
	+ <nvpair id="status-1e550a7a-16a2-453a-81c3-8d6e04919dad-fail-count-OpenVPN" name="fail-count-OpenVPN" value="2"/>
	+ </attributes>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="1e550a7a-16a2-453a-81c3-8d6e04919dad">
	+ <lrm_resources>
	+ <lrm_resource id="OpenVPN_IP" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="OpenVPN_IP_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="14:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;14:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="6" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="1f0bbac744caf5cfdfd4c698739d09c5"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_IP2" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;5:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="46" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="368d7680070c38f8f2f40ddf5a434d45"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache_IP" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_IP_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="12:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;12:5:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="4" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="a3558b7ea398267ef1ea56b21f5d840a"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_IP3" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;5:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="47" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ <lrm_rsc_op id="resource_IP3_start_0" operation="start" crm-debug-origin="do_update_resource" transition_key="32:0:5e45f285-3200-4d7f-966a-42c650d75a6c" transition_magic="0:0;32:0:5e45f285-3200-4d7f-966a-42c650d75a6c" call_id="50" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache" type="apache13" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:1;5:22:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="14" crm_feature_set="2.0" rc_code="1" op_status="4" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ <lrm_rsc_op id="Apache_stop_0" operation="stop" crm-debug-origin="build_active_RAs" transition_key="2:23:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;2:23:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="15" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummyweb" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummyweb_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;5:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="49" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummy" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="5:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;5:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="48" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ <lrm_rsc_op id="resource_dummy_start_0" operation="start" crm-debug-origin="do_update_resource" transition_key="34:0:5e45f285-3200-4d7f-966a-42c650d75a6c" transition_magic="0:0;34:0:5e45f285-3200-4d7f-966a-42c650d75a6c" call_id="51" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op_force_restart=" state " op_restart_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ <node_state uname="biggame.ds9" ha="active" crmd="online" shutdown="0" in_ccm="true" join="member" expected="member" id="de2a634c-e405-42bb-9df2-292e36ad5970" crm-debug-origin="do_lrm_query">
	+ <transient_attributes id="de2a634c-e405-42bb-9df2-292e36ad5970">
	+ <instance_attributes id="status-de2a634c-e405-42bb-9df2-292e36ad5970">
	+ <attributes>
	+ <nvpair id="status-de2a634c-e405-42bb-9df2-292e36ad5970-probe_complete" name="probe_complete" value="true"/>
	+ </attributes>
	+ </instance_attributes>
	+ </transient_attributes>
	+ <lrm id="de2a634c-e405-42bb-9df2-292e36ad5970">
	+ <lrm_resources>
	+ <lrm_resource id="resource_IP2" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="9:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;9:74:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="34" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="368d7680070c38f8f2f40ddf5a434d45"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache_IP" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_IP_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="8:26:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;8:26:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="4" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="a3558b7ea398267ef1ea56b21f5d840a"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_IP3" type="IPaddr" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_IP3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="9:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;9:75:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="35" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="87f29a4a3d4a9a7ad6eadc46f37c1f64"/>
	+ </lrm_resource>
	+ <lrm_resource id="Apache" type="apache13" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="Apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="9:26:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:1;9:26:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="5" crm_feature_set="2.0" rc_code="1" op_status="4" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ <lrm_rsc_op id="Apache_stop_0" operation="stop" crm-debug-origin="build_active_RAs" transition_key="1:27:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="0:0;1:27:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="6" crm_feature_set="2.0" rc_code="0" op_status="0" interval="0" op_digest="353bb677b42be27312c9f093a30268be"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummyweb" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummyweb_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="9:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;9:83:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="37" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ <lrm_resource id="resource_dummy" type="Dummy" class="ocf" provider="heartbeat">
	+ <lrm_rsc_op id="resource_dummy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" transition_key="9:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" transition_magic="4:7;9:79:33f424d2-dbce-4c8b-a19c-ce4da566c9d7" call_id="36" crm_feature_set="2.0" rc_code="7" op_status="4" interval="0" op_digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	+ </lrm_resource>
	+ </lrm_resources>
	+ </lrm>
	+ </node_state>
	+ </status>
	+ </cib>
	+
	diff --git a/crm/pengine/testcases/coloc-group.dot b/crm/pengine/testcases/coloc-group.dot
	index f3562e2088..38648c3f56 100644
	--- a/crm/pengine/testcases/coloc-group.dot
	+++ b/crm/pengine/testcases/coloc-group.dot
	@@ -1,42 +1,42 @@
	digraph "g" {
	-"group1_running_0" [ style=bold color="green" fontcolor="orange" ]
	-"group1_start_0" -> "group1_running_0" [ style = bold]
	+"group1_running_0" [ style=dashed color="red" fontcolor="orange" ]
	+"group1_start_0" -> "group1_running_0" [ style = dashed]
	"group1_start_0" -> "rsc2_start_0 node2" [ style = bold]
	"group1_start_0" [ style=bold color="green" fontcolor="orange" ]
	"probe_complete node1" -> "probe_complete" [ style = bold]
	"probe_complete node1" [ style=bold color="green" fontcolor="black" ]
	"probe_complete node2" -> "probe_complete" [ style = bold]
	"probe_complete node2" [ style=bold color="green" fontcolor="black" ]
	"probe_complete node3" -> "probe_complete" [ style = bold]
	"probe_complete node3" [ style=bold color="green" fontcolor="black" ]
	"probe_complete" -> "group1_start_0" [ style = bold]
	"probe_complete" -> "rsc1_start_0 node2" [ style = bold]
	"probe_complete" [ style=bold color="green" fontcolor="orange" ]
	"rsc1_monitor_0 node1" -> "probe_complete node1" [ style = bold]
	"rsc1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ]
	"rsc1_monitor_0 node2" -> "probe_complete node2" [ style = bold]
	"rsc1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ]
	"rsc1_monitor_0 node3" -> "probe_complete node3" [ style = bold]
	"rsc1_monitor_0 node3" [ style=bold color="green" fontcolor="black" ]
	"rsc1_start_0 node2" [ style=bold color="green" fontcolor="black" ]
	"rsc2_monitor_0 node2" -> "probe_complete node2" [ style = bold]
	"rsc2_monitor_0 node2" [ style=bold color="green" fontcolor="black" ]
	-"rsc2_start_0 node2" -> "group1_running_0" [ style = bold]
	+"rsc2_start_0 node2" -> "group1_running_0" [ style = dashed]
	"rsc2_start_0 node2" -> "rsc3_start_0 <none>" [ style = dashed]
	"rsc2_start_0 node2" [ style=bold color="green" fontcolor="black" ]
	"rsc3_monitor_0 node1" -> "probe_complete node1" [ style = bold]
	"rsc3_monitor_0 node1" [ style=bold color="green" fontcolor="black" ]
	"rsc3_monitor_0 node3" -> "probe_complete node3" [ style = bold]
	"rsc3_monitor_0 node3" [ style=bold color="green" fontcolor="black" ]
	"rsc3_start_0 <none>" -> "group1_running_0" [ style = dashed]
	"rsc3_start_0 <none>" -> "rsc4_start_0 <none>" [ style = dashed]
	"rsc3_start_0 <none>" [ style=dashed color="red" fontcolor="black" ]
	"rsc4_monitor_0 node1" -> "probe_complete node1" [ style = bold]
	"rsc4_monitor_0 node1" [ style=bold color="green" fontcolor="black" ]
	"rsc4_monitor_0 node2" -> "probe_complete node2" [ style = bold]
	"rsc4_monitor_0 node2" [ style=bold color="green" fontcolor="black" ]
	"rsc4_monitor_0 node3" -> "probe_complete node3" [ style = bold]
	"rsc4_monitor_0 node3" [ style=bold color="green" fontcolor="black" ]
	"rsc4_start_0 <none>" -> "group1_running_0" [ style = dashed]
	"rsc4_start_0 <none>" [ style=dashed color="red" fontcolor="black" ]
	}
	diff --git a/crm/pengine/testcases/coloc-group.exp b/crm/pengine/testcases/coloc-group.exp
	index 276d16aa02..1c442e266e 100644
	--- a/crm/pengine/testcases/coloc-group.exp
	+++ b/crm/pengine/testcases/coloc-group.exp
	@@ -1,209 +1,194 @@
	<transition_graph cluster-delay="60s" transition_id="0">
	<synapse id="0">
	<action_set>
	<rsc_op id="3" operation="monitor" operation_key="rsc1_monitor_0" on_node="node1" on_node_uuid="node1">
	<primitive id="rsc1" long-id="rsc1" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	<synapse id="1">
	<action_set>
	<rsc_op id="7" operation="monitor" operation_key="rsc1_monitor_0" on_node="node2" on_node_uuid="node2">
	<primitive id="rsc1" long-id="rsc1" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	<synapse id="2">
	<action_set>
	<rsc_op id="11" operation="monitor" operation_key="rsc1_monitor_0" on_node="node3" on_node_uuid="node3">
	<primitive id="rsc1" long-id="rsc1" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	<synapse id="3">
	<action_set>
	<rsc_op id="15" operation="start" operation_key="rsc1_start_0" on_node="node2" on_node_uuid="node2">
	<primitive id="rsc1" long-id="rsc1" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<pseudo_event id="1" operation="probe_complete" operation_key="probe_complete"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="4">
	<action_set>
	<pseudo_event id="17" operation="start" operation_key="group1_start_0">
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	</pseudo_event>
	</action_set>
	<inputs>
	<trigger>
	<pseudo_event id="1" operation="probe_complete" operation_key="probe_complete"/>
	</trigger>
	</inputs>
	</synapse>
	<synapse id="5">
	- <action_set>
	- <pseudo_event id="18" operation="running" operation_key="group1_running_0">
	- <attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	- </pseudo_event>
	- </action_set>
	- <inputs>
	- <trigger>
	- <rsc_op id="16" operation="start" operation_key="rsc2_start_0" on_node="node2" on_node_uuid="node2"/>
	- </trigger>
	- <trigger>
	- <pseudo_event id="17" operation="start" operation_key="group1_start_0"/>
	- </trigger>
	- </inputs>
	- </synapse>
	- <synapse id="6">
	<action_set>
	<rsc_op id="8" operation="monitor" operation_key="rsc2_monitor_0" on_node="node2" on_node_uuid="node2">
	<primitive id="rsc2" long-id="group1:rsc2" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="7">
	+ <synapse id="6">
	<action_set>
	<rsc_op id="16" operation="start" operation_key="rsc2_start_0" on_node="node2" on_node_uuid="node2">
	<primitive id="rsc2" long-id="group1:rsc2" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000"/>
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<pseudo_event id="17" operation="start" operation_key="group1_start_0"/>
	</trigger>
	</inputs>
	</synapse>
	- <synapse id="8">
	+ <synapse id="7">
	<action_set>
	<rsc_op id="4" operation="monitor" operation_key="rsc3_monitor_0" on_node="node1" on_node_uuid="node1">
	<primitive id="rsc3" long-id="group1:rsc3" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="9">
	+ <synapse id="8">
	<action_set>
	<rsc_op id="12" operation="monitor" operation_key="rsc3_monitor_0" on_node="node3" on_node_uuid="node3">
	<primitive id="rsc3" long-id="group1:rsc3" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="10">
	+ <synapse id="9">
	<action_set>
	<rsc_op id="5" operation="monitor" operation_key="rsc4_monitor_0" on_node="node1" on_node_uuid="node1">
	<primitive id="rsc4" long-id="group1:rsc4" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="11">
	+ <synapse id="10">
	<action_set>
	<rsc_op id="9" operation="monitor" operation_key="rsc4_monitor_0" on_node="node2" on_node_uuid="node2">
	<primitive id="rsc4" long-id="group1:rsc4" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="12">
	+ <synapse id="11">
	<action_set>
	<rsc_op id="13" operation="monitor" operation_key="rsc4_monitor_0" on_node="node3" on_node_uuid="node3">
	<primitive id="rsc4" long-id="group1:rsc4" class="heartbeat" type="apache"/>
	<attributes crm_feature_set="2.0" CRM_meta_timeout="20000" CRM_meta_op_target_rc="7"/>
	</rsc_op>
	</action_set>
	<inputs/>
	</synapse>
	- <synapse id="13">
	+ <synapse id="12">
	<action_set>
	<pseudo_event id="1" operation="probe_complete" operation_key="probe_complete">
	<attributes crm_feature_set="2.0"/>
	</pseudo_event>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="2" operation="probe_complete" operation_key="probe_complete" on_node="node1" on_node_uuid="node1"/>
	</trigger>
	<trigger>
	<rsc_op id="6" operation="probe_complete" operation_key="probe_complete" on_node="node2" on_node_uuid="node2"/>
	</trigger>
	<trigger>
	<rsc_op id="10" operation="probe_complete" operation_key="probe_complete" on_node="node3" on_node_uuid="node3"/>
	</trigger>
	</inputs>
	</synapse>
	- <synapse id="14" priority="1000000">
	+ <synapse id="13" priority="1000000">
	<action_set>
	<rsc_op id="2" operation="probe_complete" operation_key="probe_complete" on_node="node1" on_node_uuid="node1">
	<attributes crm_feature_set="2.0" CRM_meta_op_no_wait="true"/>
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="3" operation="monitor" operation_key="rsc1_monitor_0" on_node="node1" on_node_uuid="node1"/>
	</trigger>
	<trigger>
	<rsc_op id="4" operation="monitor" operation_key="rsc3_monitor_0" on_node="node1" on_node_uuid="node1"/>
	</trigger>
	<trigger>
	<rsc_op id="5" operation="monitor" operation_key="rsc4_monitor_0" on_node="node1" on_node_uuid="node1"/>
	</trigger>
	</inputs>
	</synapse>
	- <synapse id="15" priority="1000000">
	+ <synapse id="14" priority="1000000">
	<action_set>
	<rsc_op id="6" operation="probe_complete" operation_key="probe_complete" on_node="node2" on_node_uuid="node2">
	<attributes crm_feature_set="2.0" CRM_meta_op_no_wait="true"/>
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="7" operation="monitor" operation_key="rsc1_monitor_0" on_node="node2" on_node_uuid="node2"/>
	</trigger>
	<trigger>
	<rsc_op id="8" operation="monitor" operation_key="rsc2_monitor_0" on_node="node2" on_node_uuid="node2"/>
	</trigger>
	<trigger>
	<rsc_op id="9" operation="monitor" operation_key="rsc4_monitor_0" on_node="node2" on_node_uuid="node2"/>
	</trigger>
	</inputs>
	</synapse>
	- <synapse id="16" priority="1000000">
	+ <synapse id="15" priority="1000000">
	<action_set>
	<rsc_op id="10" operation="probe_complete" operation_key="probe_complete" on_node="node3" on_node_uuid="node3">
	<attributes crm_feature_set="2.0" CRM_meta_op_no_wait="true"/>
	</rsc_op>
	</action_set>
	<inputs>
	<trigger>
	<rsc_op id="11" operation="monitor" operation_key="rsc1_monitor_0" on_node="node3" on_node_uuid="node3"/>
	</trigger>
	<trigger>
	<rsc_op id="12" operation="monitor" operation_key="rsc3_monitor_0" on_node="node3" on_node_uuid="node3"/>
	</trigger>
	<trigger>
	<rsc_op id="13" operation="monitor" operation_key="rsc4_monitor_0" on_node="node3" on_node_uuid="node3"/>
	</trigger>
	</inputs>
	</synapse>
	</transition_graph>

	diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
	index afdb61a8a1..8dfe27e735 100644
	--- a/include/crm/pengine/status.h
	+++ b/include/crm/pengine/status.h
	@@ -1,223 +1,224 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef PENGINE_STATUS__H
	#define PENGINE_STATUS__H

	#include <glib.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/common.h>

	typedef struct node_s node_t;
	typedef struct action_s action_t;
	typedef struct resource_s resource_t;

	typedef enum no_quorum_policy_e {
	no_quorum_freeze,
	no_quorum_stop,
	no_quorum_ignore
	} no_quorum_policy_t;

	enum node_type {
	node_ping,
	node_member
	};

	enum pe_restart {
	pe_restart_restart,
	pe_restart_ignore
	};

	typedef struct pe_working_set_s
	{
	crm_data_t *input;
	ha_time_t *now;

	/* options extracted from the input */
	char *transition_idle_timeout;
	char *dc_uuid;
	node_t *dc_node;
	gboolean have_quorum;
	gboolean stonith_enabled;
	const char *stonith_action;
	gboolean symmetric_cluster;
	gboolean is_managed_default;

	gboolean start_failure_fatal;

	gboolean remove_after_stop;
	gboolean stop_rsc_orphans;
	gboolean stop_action_orphans;

	int default_resource_stickiness;
	int default_resource_fail_stickiness;
	no_quorum_policy_t no_quorum_policy;

	GHashTable *config_hash;

	GListPtr nodes;
	GListPtr resources;
	GListPtr placement_constraints;
	GListPtr ordering_constraints;
	GListPtr colocation_constraints;

	GListPtr actions;
	crm_data_t *failed;

	/* stats */
	int num_synapse;
	int max_valid_nodes;
	int order_id;
	int action_id;

	/* final output */
	crm_data_t *graph;

	} pe_working_set_t;

	struct node_shared_s {
	const char *id;
	const char *uname;
	gboolean online;
	gboolean standby;
	gboolean unclean;
	gboolean shutdown;
	gboolean expected_up;
	gboolean is_dc;
	int num_resources;
	GListPtr running_rsc; /* resource_t* */
	GListPtr allocated_rsc; /* resource_t* */

	GHashTable attrs; / char* => char* */
	enum node_type type;
	};

	struct node_s {
	int weight;
	gboolean fixed;
	int count;
	struct node_shared_s *details;
	};

	#include <crm/pengine/complex.h>

	struct resource_s {
	char *id;
	char *clone_name;
	char *long_name;
	crm_data_t *xml;
	crm_data_t *ops_xml;

	resource_t *parent;
	void *variant_opaque;
	enum pe_obj_types variant;
	resource_object_functions_t *fns;
	resource_alloc_functions_t *cmds;

	enum rsc_recovery_type recovery_type;
	enum pe_restart restart_type;

	int priority;
	int stickiness;
	int sort_index;
	int fail_stickiness;
	int effective_priority;

	gboolean notify;
	gboolean is_managed;
	gboolean can_migrate;
	gboolean starting;
	gboolean stopping;
	gboolean runnable;
	gboolean provisional;
	gboolean globally_unique;
	gboolean is_allocating;
	gboolean is_merging;

	gboolean failed;
	gboolean start_pending;
	+ gboolean shutdown;

	gboolean orphan;

	GListPtr rsc_cons_lhs; /* rsc_colocation_t* */
	GListPtr rsc_cons; /* rsc_colocation_t* */
	GListPtr rsc_location; /* rsc_to_node_t* */
	GListPtr actions; /* action_t* */

	node_t *allocated_to;
	GListPtr running_on; /* node_t* */
	GListPtr known_on; /* node_t* */
	GListPtr allowed_nodes; /* node_t* */

	enum rsc_role_e role;
	enum rsc_role_e next_role;

	GHashTable *meta;
	GHashTable *parameters;

	GListPtr children; /* resource_t* */
	};

	struct action_s
	{
	int id;
	int priority;
	resource_t *rsc;
	void *rsc_opaque;
	node_t *node;
	char *task;

	char *uuid;
	crm_data_t *op_entry;

	gboolean pseudo;
	gboolean runnable;
	gboolean optional;
	gboolean failure_is_fatal;
	gboolean implied_by_stonith;
	gboolean allow_reload_conversion;

	enum rsc_start_requirement needs;
	enum action_fail_response on_fail;
	enum rsc_role_e fail_role;

	gboolean dumped;
	gboolean processed;

	action_t *pre_notify;
	action_t *pre_notified;
	action_t *post_notify;
	action_t *post_notified;

	int seen_count;

	GHashTable *meta;
	GHashTable *extra;
	GHashTable notify_keys; / do NOT free */

	GListPtr actions_before; /* action_warpper_t* */
	GListPtr actions_after; /* action_warpper_t* */
	};

	gboolean cluster_status(pe_working_set_t *data_set);
	extern void set_working_set_defaults(pe_working_set_t *data_set);
	extern void cleanup_calculations(pe_working_set_t *data_set);
	extern resource_t pe_find_resource(GListPtr rsc_list, const char id_rh);
	extern node_t pe_find_node(GListPtr node_list, const char uname);
	extern node_t pe_find_node_id(GListPtr node_list, const char id);

	#endif

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:07 PM (1 d, 2 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002484
Default Alt Text: (122 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions