No OneTemporary
Actions

Size

379 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/pcmki/pcmki_sched_utils.h b/include/pcmki/pcmki_sched_utils.h
	index e49263fef6..751232f520 100644
	--- a/include/pcmki/pcmki_sched_utils.h
	+++ b/include/pcmki/pcmki_sched_utils.h
	@@ -1,102 +1,97 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PENGINE_AUTILS__H
	# define PENGINE_AUTILS__H

	#include <stdbool.h> // bool
	#include <glib.h> // GList, GHashTable, gboolean, guint
	#include <crm/lrmd.h> // lrmd_event_data_t
	#include <crm/cib.h> // cib_t
	#include <crm/pengine/pe_types.h>
	#include <crm/pengine/internal.h>
	#include <pcmki/pcmki_scheduler.h>
	#include <pcmki/pcmki_transition.h>
	#include <pacemaker.h>

	/* Constraint helper functions */
	pcmk__colocation_t invert_constraint(pcmk__colocation_t constraint);

	pe__location_t copy_constraint(pe__location_t constraint);

	pe__location_t rsc2node_new(const char id, pe_resource_t *rsc, int weight,
	const char discovery_mode, pe_node_t node,
	pe_working_set_t *data_set);

	-void pcmk__new_colocation(const char id, const char node_attr, int score,
	- pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	- const char state_lh, const char state_rh,
	- bool influence, pe_working_set_t *data_set);
	-
	GHashTable pcmk__copy_node_table(GHashTable nodes);
	GList pcmk__copy_node_list(const GList list, bool reset);
	GList sort_nodes_by_weight(GList nodes, pe_node_t *active_node,
	pe_working_set_t *data_set);

	extern gboolean can_run_resources(const pe_node_t * node);
	extern gboolean native_assign_node(pe_resource_t rsc, pe_node_t chosen,
	gboolean force);
	void native_deallocate(pe_resource_t * rsc);

	extern void log_action(unsigned int log_level, const char *pre_text,
	pe_action_t * action, gboolean details);

	gboolean can_run_any(GHashTable * nodes);
	pe_resource_t find_compatible_child(pe_resource_t local_child,
	pe_resource_t *rsc, enum rsc_role_e filter,
	gboolean current,
	pe_working_set_t *data_set);
	pe_resource_t find_compatible_child_by_node(pe_resource_t local_child, pe_node_t * local_node, pe_resource_t * rsc,
	enum rsc_role_e filter, gboolean current);
	gboolean is_child_compatible(pe_resource_t child_rsc, pe_node_t local_node, enum rsc_role_e filter, gboolean current);
	bool assign_node(pe_resource_t * rsc, pe_node_t * node, gboolean force);
	enum pe_action_flags summary_action_flags(pe_action_t * action, GList children, pe_node_t node);
	enum action_tasks clone_child_action(pe_action_t * action);
	int copies_per_node(pe_resource_t * rsc);

	enum filter_colocation_res {
	influence_nothing = 0,
	influence_rsc_location,
	influence_rsc_priority,
	};

	extern enum filter_colocation_res
	filter_colocation_constraint(pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
	pcmk__colocation_t *constraint, gboolean preview);

	extern int compare_capacity(const pe_node_t * node1, const pe_node_t * node2);
	extern void calculate_utilization(GHashTable * current_utilization,
	GHashTable * utilization, gboolean plus);

	extern void process_utilization(pe_resource_t * rsc, pe_node_t ** prefer, pe_working_set_t * data_set);
	pe_action_t create_pseudo_resource_op(pe_resource_t rsc, const char task, bool optional, bool runnable, pe_working_set_t data_set);
	pe_action_t pe_cancel_op(pe_resource_t rsc, const char *name,
	guint interval_ms, pe_node_t *node,
	pe_working_set_t *data_set);
	pe_action_t sched_shutdown_op(pe_node_t node, pe_working_set_t *data_set);

	xmlNode pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t *event,
	const char *caller_version, int target_rc,
	const char node, const char origin,
	int level);

	# define LOAD_STOPPED "load_stopped"

	void modify_configuration(pe_working_set_t data_set, cib_t cib,
	pcmk_injections_t *injections);

	enum transition_status run_simulation(pe_working_set_t * data_set, cib_t cib, GList op_fail_list);

	pcmk__output_t *pcmk__new_logger(void);

	bool pcmk__threshold_reached(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set,
	pe_resource_t **failed);

	#endif
	diff --git a/lib/pacemaker/Makefile.am b/lib/pacemaker/Makefile.am
	index 314497ff6e..78703bc60b 100644
	--- a/lib/pacemaker/Makefile.am
	+++ b/lib/pacemaker/Makefile.am
	@@ -1,54 +1,55 @@
	#
	# Copyright 2004-2021 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	include $(top_srcdir)/mk/common.mk

	AM_CPPFLAGS += -I$(top_builddir) -I$(top_srcdir)

	noinst_HEADERS = libpacemaker_private.h

	## libraries
	lib_LTLIBRARIES = libpacemaker.la

	## SOURCES

	libpacemaker_la_LDFLAGS = -version-info 3:1:2

	libpacemaker_la_CFLAGS = $(CFLAGS_HARDENED_LIB)
	libpacemaker_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB)

	libpacemaker_la_LIBADD = $(top_builddir)/lib/pengine/libpe_status.la \
	$(top_builddir)/lib/cib/libcib.la \
	$(top_builddir)/lib/lrmd/liblrmd.la \
	$(top_builddir)/lib/common/libcrmcommon.la
	# -L$(top_builddir)/lib/pils -lpils -export-dynamic -module -avoid-version
	# Use += rather than backlashed continuation lines for parsing by bumplibs
	libpacemaker_la_SOURCES =
	libpacemaker_la_SOURCES += pcmk_cluster_queries.c
	libpacemaker_la_SOURCES += pcmk_fence.c
	libpacemaker_la_SOURCES += pcmk_graph_consumer.c
	libpacemaker_la_SOURCES += pcmk_graph_logging.c
	libpacemaker_la_SOURCES += pcmk_graph_producer.c
	libpacemaker_la_SOURCES += pcmk_output.c
	libpacemaker_la_SOURCES += pcmk_output_utils.c
	libpacemaker_la_SOURCES += pcmk_resource.c
	libpacemaker_la_SOURCES += pcmk_sched_allocate.c
	libpacemaker_la_SOURCES += pcmk_sched_bundle.c
	libpacemaker_la_SOURCES += pcmk_sched_clone.c
	+libpacemaker_la_SOURCES += pcmk_sched_colocation.c
	libpacemaker_la_SOURCES += pcmk_sched_constraints.c
	libpacemaker_la_SOURCES += pcmk_sched_fencing.c
	libpacemaker_la_SOURCES += pcmk_sched_group.c
	libpacemaker_la_SOURCES += pcmk_sched_messages.c
	libpacemaker_la_SOURCES += pcmk_sched_native.c
	libpacemaker_la_SOURCES += pcmk_sched_notif.c
	libpacemaker_la_SOURCES += pcmk_sched_promotable.c
	libpacemaker_la_SOURCES += pcmk_sched_transition.c
	libpacemaker_la_SOURCES += pcmk_sched_utilization.c
	libpacemaker_la_SOURCES += pcmk_sched_utils.c
	libpacemaker_la_SOURCES += pcmk_simulate.c
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index ee64ec0b9c..dabe11fc4d 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,37 +1,65 @@
	/*
	* Copyright 2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__LIBPACEMAKER_PRIVATE__H
	# define PCMK__LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <crm/pengine/pe_types.h> // pe_action_t, pe_node_t, pe_working_set_t

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(pe_resource_t rsc, pe_node_t node,
	pe_action_t *action, enum pe_ordering order,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pe_node_t node, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__is_unfence_device(const pe_resource_t *rsc,
	const pe_working_set_t *data_set);

	+G_GNUC_INTERNAL
	+pe_resource_t pcmk__find_constraint_resource(GList rsc_list, const char *id);
	+
	+G_GNUC_INTERNAL
	+xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	+ pe_working_set_t *data_set);
	+
	+G_GNUC_INTERNAL
	+gboolean pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	+ pe_resource_t rsc, pe_tag_t tag);
	+
	+G_GNUC_INTERNAL
	+gboolean pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	+ gboolean convert_rsc, pe_working_set_t *data_set);
	+
	+G_GNUC_INTERNAL
	+void pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set);
	+
	+G_GNUC_INTERNAL
	+void pcmk__new_colocation(const char id, const char node_attr, int score,
	+ pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	+ const char state_lh, const char state_rh,
	+ bool influence, pe_working_set_t *data_set);
	+
	+G_GNUC_INTERNAL
	+void pcmk__block_colocated_starts(pe_action_t *action,
	+ pe_working_set_t *data_set);
	+
	#endif // PCMK__LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
	index a52a2bb872..e91cda0107 100644
	--- a/lib/pacemaker/pcmk_graph_producer.c
	+++ b/lib/pacemaker/pcmk_graph_producer.c
	@@ -1,1846 +1,1781 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	-void update_colo_start_chain(pe_action_t action, pe_working_set_t data_set);
	+#include "libpacemaker_private.h"
	+
	gboolean rsc_update_action(pe_action_t * first, pe_action_t * then, enum pe_ordering type);

	static enum pe_action_flags
	get_action_flags(pe_action_t * action, pe_node_t * node)
	{
	enum pe_action_flags flags = action->flags;

	if (action->rsc) {
	flags = action->rsc->cmds->action_flags(action, NULL);

	if (pe_rsc_is_clone(action->rsc) && node) {

	/* We only care about activity on $node */
	enum pe_action_flags clone_flags = action->rsc->cmds->action_flags(action, node);

	/* Go to great lengths to ensure the correct value for pe_action_runnable...
	*
	* If we are a clone, then for _ordering_ constraints, it's only relevant
	* if we are runnable _anywhere_.
	*
	* This only applies to _runnable_ though, and only for ordering constraints.
	* If this function is ever used during colocation, then we'll need additional logic
	*
	* Not very satisfying, but it's logical and appears to work well.
	*/
	if (!pcmk_is_set(clone_flags, pe_action_runnable)
	&& pcmk_is_set(flags, pe_action_runnable)) {

	pe__set_raw_action_flags(clone_flags, action->rsc->id,
	pe_action_runnable);
	}
	flags = clone_flags;
	}
	}
	return flags;
	}

	static char *
	convert_non_atomic_uuid(char old_uuid, pe_resource_t rsc, gboolean allow_notify,
	gboolean free_original)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *raw_task = NULL;
	int task = no_action;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing %s", old_uuid);
	if (old_uuid == NULL) {
	return NULL;

	} else if (strstr(old_uuid, "notify") != NULL) {
	goto done; /* no conversion */

	} else if (rsc->variant < pe_group) {
	goto done; /* no conversion */
	}

	CRM_ASSERT(parse_op_key(old_uuid, &rid, &raw_task, &interval_ms));
	if (interval_ms > 0) {
	goto done; /* no conversion */
	}

	task = text2task(raw_task);
	switch (task) {
	case stop_rsc:
	case start_rsc:
	case action_notify:
	case action_promote:
	case action_demote:
	break;
	case stopped_rsc:
	case started_rsc:
	case action_notified:
	case action_promoted:
	case action_demoted:
	task--;
	break;
	case monitor_rsc:
	case shutdown_crm:
	case stonith_node:
	task = no_action;
	break;
	default:
	crm_err("Unknown action: %s", raw_task);
	task = no_action;
	break;
	}

	if (task != no_action) {
	if (pcmk_is_set(rsc->flags, pe_rsc_notify) && allow_notify) {
	uuid = pcmk__notify_key(rid, "confirmed-post", task2text(task + 1));

	} else {
	uuid = pcmk__op_key(rid, task2text(task + 1), 0);
	}
	pe_rsc_trace(rsc, "Converted %s -> %s", old_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(old_uuid);
	}

	if (free_original) {
	free(old_uuid);
	}

	free(raw_task);
	free(rid);
	return uuid;
	}

	static pe_action_t *
	rsc_expand_action(pe_action_t * action)
	{
	gboolean notify = FALSE;
	pe_action_t *result = action;
	pe_resource_t *rsc = action->rsc;

	if (rsc == NULL) {
	return action;
	}

	if ((rsc->parent == NULL)
	\|\| (pe_rsc_is_clone(rsc) && (rsc->parent->variant == pe_container))) {
	/* Only outermost resources have notification actions.
	* The exception is those in bundles.
	*/
	notify = pcmk_is_set(rsc->flags, pe_rsc_notify);
	}

	if (rsc->variant >= pe_group) {
	/* Expand 'start' -> 'started' */
	char *uuid = NULL;

	uuid = convert_non_atomic_uuid(action->uuid, rsc, notify, FALSE);
	if (uuid) {
	pe_rsc_trace(rsc, "Converting %s to %s %d", action->uuid, uuid,
	pcmk_is_set(rsc->flags, pe_rsc_notify));
	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_err("Couldn't expand %s to %s in %s", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	}
	return result;
	}

	static enum pe_graph_flags
	graph_update_action(pe_action_t * first, pe_action_t * then, pe_node_t * node,
	enum pe_action_flags first_flags, enum pe_action_flags then_flags,
	pe_action_wrapper_t order, pe_working_set_t data_set)
	{
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_ordering type = order->type;

	/* TODO: Do as many of these in parallel as possible */

	if (pcmk_is_set(type, pe_order_implies_then_on_node)) {
	/* Normally we want the _whole_ 'then' clone to
	* restart if 'first' is restarted, so then->node is
	* needed.
	*
	* However for unfencing, we want to limit this to
	* instances on the same node as 'first' (the
	* unfencing operation), so first->node is supplied.
	*
	* Swap the node, from then on we can can treat it
	* like any other 'pe_order_implies_then'
	*/

	pe__clear_order_flags(type, pe_order_implies_then_on_node);
	pe__set_order_flags(type, pe_order_implies_then);
	node = first->node;
	pe_rsc_trace(then->rsc,
	"%s then %s: mapped pe_order_implies_then_on_node to "
	"pe_order_implies_then on %s",
	first->uuid, then->uuid, node->details->uname);
	}

	if (type & pe_order_implies_then) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags & pe_action_optional, pe_action_optional,
	pe_order_implies_then, data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)) {
	pe__clear_action_flags(then, pe_action_optional);
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if ((type & pe_order_restart) && then->rsc) {
	enum pe_action_flags restart = (pe_action_optional \| pe_action_runnable);

	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, restart,
	pe_order_restart, data_set);
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_restart",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_implies_first) {
	if (first->rsc) {
	changed \|= first->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional, pe_order_implies_first,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(first->flags, pe_action_runnable)) {
	pe__clear_action_flags(first, pe_action_runnable);
	pe__set_graph_flags(changed, first, pe_graph_updated_first);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_promoted_implies_first) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags & pe_action_optional, pe_action_optional,
	pe_order_promoted_implies_first, data_set);
	}
	pe_rsc_trace(then->rsc,
	"%s then %s: %s after pe_order_promoted_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_one_or_more) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_one_or_more,
	data_set);

	} else if (pcmk_is_set(first_flags, pe_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__set_action_flags(then, pe_action_runnable);
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_one_or_more",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (then->rsc && pcmk_is_set(type, pe_order_probe)) {
	if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& (first->rsc->running_on != NULL)) {

	pe_rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	type = pe_order_none;
	order->type = pe_order_none;

	} else {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_runnable_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_runnable_left) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_runnable_left,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__clear_action_flags(then, pe_action_runnable);
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_runnable_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_implies_first_migratable) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional,
	pe_order_implies_first_migratable, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after "
	"pe_order_implies_first_migratable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_pseudo_left) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_optional, pe_order_pseudo_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_pseudo_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_optional) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_optional, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_optional",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (type & pe_order_asymmetrical) {
	if (then->rsc) {
	changed \|= then->rsc->cmds->update_actions(first, then, node,
	first_flags, pe_action_runnable, pe_order_asymmetrical,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_asymmetrical",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if ((first->flags & pe_action_runnable) && (type & pe_order_implies_then_printed)
	&& (first_flags & pe_action_optional) == 0) {
	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pe__set_action_flags(then, pe_action_print_always);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(type, pe_order_implies_first_printed)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pe__set_action_flags(first, pe_action_print_always);
	// Don't bother marking 'first' as changed just for this
	}

	if ((type & pe_order_implies_then
	\|\| type & pe_order_implies_first
	\|\| type & pe_order_restart)
	&& first->rsc
	&& pcmk__str_eq(first->task, RSC_STOP, pcmk__str_casei)
	&& !pcmk_is_set(first->rsc->flags, pe_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pe_rsc_block)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {

	if (pcmk_is_set(then->flags, pe_action_runnable)) {
	pe__clear_action_flags(then, pe_action_runnable);
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	-static void
	-mark_start_blocked(pe_resource_t rsc, pe_resource_t reason,
	- pe_working_set_t *data_set)
	-{
	- GList *gIter = rsc->actions;
	- char *reason_text = crm_strdup_printf("colocation with %s", reason->id);
	-
	- for (; gIter != NULL; gIter = gIter->next) {
	- pe_action_t action = (pe_action_t ) gIter->data;
	-
	- if (!pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	- continue;
	- }
	- if (pcmk_is_set(action->flags, pe_action_runnable)) {
	- pe__clear_action_flags(action, pe_action_runnable);
	- pe_action_set_reason(action, reason_text, false);
	- update_colo_start_chain(action, data_set);
	- update_action(action, data_set);
	- }
	- }
	- free(reason_text);
	-}
	-
	-void
	-update_colo_start_chain(pe_action_t action, pe_working_set_t data_set)
	-{
	- GList *gIter = NULL;
	- pe_resource_t *rsc = NULL;
	-
	- if (!pcmk_is_set(action->flags, pe_action_runnable)
	- && pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	-
	- rsc = uber_parent(action->rsc);
	- if (rsc->parent) {
	- /* For bundles, uber_parent() returns the clone, not the bundle, so
	- * the existence of rsc->parent implies this is a bundle.
	- * In this case, we need the bundle resource, so that we can check
	- * if all containers are stopped/stopping.
	- */
	- rsc = rsc->parent;
	- }
	- }
	-
	- if (rsc == NULL \|\| rsc->rsc_cons_lhs == NULL) {
	- return;
	- }
	-
	- /* if rsc has children, all the children need to have start set to
	- * unrunnable before we follow the colo chain for the parent. */
	- for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	- pe_resource_t child = (pe_resource_t )gIter->data;
	- pe_action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);
	- if ((start == NULL) \|\| pcmk_is_set(start->flags, pe_action_runnable)) {
	- return;
	- }
	- }
	-
	- for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	- pcmk__colocation_t colocate_with = (pcmk__colocation_t ) gIter->data;
	-
	- if (colocate_with->score == INFINITY) {
	- mark_start_blocked(colocate_with->rsc_lh, action->rsc, data_set);
	- }
	- }
	-}
	-
	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pe_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	gboolean
	update_action(pe_action_t then, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	enum pe_graph_flags changed = pe_graph_none;
	int last_flags = then->flags;

	pe_rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	/* initialize current known runnable before actions to 0
	* from here as graph_update_action is called for each of
	* then's before actions, this number will increment as
	* runnable 'first' actions are encountered */
	then->runnable_before = 0;

	/* for backwards compatibility with previous options that use
	* the 'requires_any' flag, initialize required to 1 if it is
	* not set. */
	if (then->required_runnable_before == 0) {
	then->required_runnable_before = 1;
	}
	pe__clear_action_flags(then, pe_action_runnable);
	/* We are relying on the pe_order_one_or_more clause of
	* graph_update_action(), called as part of the:
	*
	* 'if (first == other->action)'
	*
	* block below, to set this back if appropriate
	*/
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;
	pe_action_t *first = other->action;

	pe_node_t *then_node = then->node;
	pe_node_t *first_node = first->node;

	enum pe_action_flags then_flags = 0;
	enum pe_action_flags first_flags = 0;

	if (first->rsc && first->rsc->variant == pe_group && pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node) {
	pe_rsc_trace(first->rsc, "Found node %s for 'first' %s",
	first_node->details->uname, first->uuid);
	}
	}

	if (then->rsc && then->rsc->variant == pe_group && pcmk__str_eq(then->task, RSC_START, pcmk__str_casei)) {
	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node) {
	pe_rsc_trace(then->rsc, "Found node %s for 'then' %s",
	then_node->details->uname, then->uuid);
	}
	}
	/* Disable constraint if it only applies when on same node, but isn't */
	if (pcmk_is_set(other->type, pe_order_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& (first_node->details != then_node->details)) {

	pe_rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: not same node",
	other->action->uuid, first_node->details->uname,
	then->uuid, then_node->details->uname);
	other->type = pe_order_none;
	continue;
	}

	pe__clear_graph_flags(changed, then, pe_graph_updated_first);

	if (first->rsc && pcmk_is_set(other->type, pe_order_then_cancels_first)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pe__set_action_flags(other->action, pe_action_optional);
	if (!strcmp(first->task, CRMD_ACTION_RELOAD_AGENT)) {
	pe__clear_resource_flags(first->rsc, pe_rsc_reload);
	}
	}

	if (first->rsc && then->rsc && (first->rsc != then->rsc)
	&& (is_parent(then->rsc, first->rsc) == FALSE)) {
	first = rsc_expand_action(first);
	}
	if (first != other->action) {
	pe_rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	first_flags = get_action_flags(first, then_node);
	then_flags = get_action_flags(then, first_node);

	pe_rsc_trace(then->rsc,
	"%s then %s: type=0x%.6x filter=0x%.6x "
	"(%s %s %s on %s 0x%.6x then 0x%.6x)",
	first->uuid, then->uuid, other->type, first_flags,
	action_optional_str(first_flags),
	action_runnable_str(first_flags),
	action_type_str(first_flags), action_node_str(first),
	first->flags, then->flags);

	if (first == other->action) {
	/*
	* 'first' was not expanded (e.g. from 'start' to 'running'), which could mean it:
	* - has no associated resource,
	* - was a primitive,
	* - was pre-expanded (e.g. 'running' instead of 'start')
	*
	* The third argument here to graph_update_action() is a node which is used under two conditions:
	* - Interleaving, in which case first->node and
	* then->node are equal (and NULL)
	* - If 'then' is a clone, to limit the scope of the
	* constraint to instances on the supplied node
	*
	*/
	pe_node_t *node = then->node;
	changed \|= graph_update_action(first, then, node, first_flags,
	then_flags, other, data_set);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pe__set_graph_flags(changed, then,
	pe_graph_updated_then\|pe_graph_disable);
	}

	if (changed & pe_graph_disable) {
	pe_rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	pe__clear_graph_flags(changed, then, pe_graph_disable);
	other->type = pe_order_none;
	}

	if (changed & pe_graph_updated_first) {
	GList *lpc2 = NULL;

	crm_trace("Re-processing %s and its 'after' actions since it changed",
	first->uuid);
	for (lpc2 = first->actions_after; lpc2 != NULL; lpc2 = lpc2->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc2->data;

	update_action(other->action, data_set);
	}
	update_action(first, data_set);
	}
	}

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	if (last_flags != then->flags) {
	pe__set_graph_flags(changed, then, pe_graph_updated_then);
	} else {
	pe__clear_graph_flags(changed, then, pe_graph_updated_then);
	}
	}

	if (changed & pe_graph_updated_then) {
	crm_trace("Re-processing %s and its 'after' actions since it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {
	- update_colo_start_chain(then, data_set);
	+ pcmk__block_colocated_starts(then, data_set);
	}
	update_action(then, data_set);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;

	update_action(other->action, data_set);
	}
	}

	return FALSE;
	}

	gboolean
	shutdown_constraints(pe_node_t * node, pe_action_t * shutdown_op, pe_working_set_t * data_set)
	{
	/* add the stop to the before lists so it counts as a pre-req
	* for the shutdown
	*/
	GList *lpc = NULL;

	for (lpc = data_set->actions; lpc != NULL; lpc = lpc->next) {
	pe_action_t action = (pe_action_t ) lpc->data;

	if (action->rsc == NULL \|\| action->node == NULL) {
	continue;
	} else if (action->node->details != node->details) {
	continue;
	} else if (pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)) {
	pe_rsc_trace(action->rsc, "Skipping %s: maintenance mode", action->uuid);
	continue;
	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc, "Skipping %s: node %s is in maintenance mode",
	action->uuid, node->details->uname);
	continue;
	} else if (!pcmk__str_eq(action->task, RSC_STOP, pcmk__str_casei)) {
	continue;
	} else if (!pcmk_any_flags_set(action->rsc->flags,
	pe_rsc_managed\|pe_rsc_block)) {
	/*
	* If another action depends on this one, we may still end up blocking
	*/
	pe_rsc_trace(action->rsc, "Skipping %s: unmanaged", action->uuid);
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown on %s", action->uuid,
	node->details->uname);
	pe__clear_action_flags(action, pe_action_optional);
	custom_action_order(action->rsc, NULL, action,
	NULL, strdup(CRM_OP_SHUTDOWN), shutdown_op,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}

	return TRUE;
	}

	static pe_node_t *
	get_router_node(pe_action_t *action)
	{
	pe_node_t *began_on = NULL;
	pe_node_t *ended_on = NULL;
	bool partial_migration = FALSE;
	const char *task = action->task;

	if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)
	\|\| !pe__is_guest_or_remote_node(action->node)) {
	return NULL;
	}

	CRM_ASSERT(action->node->details->remote_rsc != NULL);

	began_on = pe__current_node(action->node->details->remote_rsc);
	ended_on = action->node->details->remote_rsc->allocated_to;
	if (action->node->details->remote_rsc
	&& (action->node->details->remote_rsc->container == NULL)
	&& action->node->details->remote_rsc->partial_migration_target) {
	partial_migration = TRUE;
	}

	if (began_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (starting)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	if (ended_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (stopping)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	if (began_on->details == ended_on->details) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (not moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* If we get here, the remote connection is moving during this transition.
	* This means some actions for resources behind the connection will get
	* routed through the cluster node the connection reource is currently on,
	* and others are routed through the cluster node the connection will end up
	* on.
	*/

	if (pcmk__str_eq(task, "notify", pcmk__str_casei)) {
	task = g_hash_table_lookup(action->meta, "notify_operation");
	}

	/*
	* Stop, demote, and migration actions must occur before the connection can
	* move (these actions are required before the remote resource can stop). In
	* this case, we know these actions have to be routed through the initial
	* cluster node the connection resource lived on before the move takes
	* place.
	*
	* The exception is a partial migration of a (non-guest) remote connection
	* resource; in that case, all actions (even these) will be ordered after
	* the connection's pseudo-start on the migration target, so the target is
	* the router node.
	*/
	if (pcmk__strcase_any_of(task, "cancel", "stop", "demote", "migrate_from",
	"migrate_to", NULL) && !partial_migration) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* Everything else (start, promote, monitor, probe, refresh,
	* clear failcount, delete, ...) must occur after the connection starts on
	* the node it is moving to.
	*/
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, XML_ATTR_UUID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pe_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Add XML with nodes that need an update of their maintenance state
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] data_set Working set for cluster
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pe_working_set_t data_set)
	{
	GList *gIter = NULL;
	xmlNode *maintenance =
	xml?create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE):NULL;
	int count = 0;

	for (gIter = data_set->nodes; gIter != NULL;
	gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	struct pe_node_shared_s *details = node->details;

	if (!pe__is_guest_or_remote_node(node)) {
	continue; /* just remote nodes need to know atm */
	}

	if (details->maintenance != details->remote_maintenance) {
	if (maintenance) {
	crm_xml_add(
	add_node_to_xml_by_id(node->details->id, maintenance),
	XML_NODE_IS_MAINTENANCE, details->maintenance?"1":"0");
	}
	count++;
	}
	}
	crm_trace("%s %d nodes to adjust maintenance-mode "
	"to transition", maintenance?"Added":"Counted", count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] data_set Working set for cluster
	*/
	void
	add_maintenance_update(pe_working_set_t *data_set)
	{
	pe_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, data_set)) {
	crm_trace("adding maintenance state update pseudo action");
	action = get_pseudo_op(CRM_OP_MAINTENANCE_NODES, data_set);
	pe__set_action_flags(action, pe_action_print_always);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	* \param[in] data_set Working set for cluster
	*/
	static void
	add_downed_nodes(xmlNode xml, const pe_action_t action,
	const pe_working_set_t *data_set)
	{
	CRM_CHECK(xml && action && action->node && data_set, return);

	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__strcase_any_of(fence, "off", "reboot", NULL)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(data_set, action->node, add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, CRMD_ACTION_STOP, pcmk__str_casei)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pe_action_t *input;
	gboolean migrating = FALSE;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pe_action_wrapper_t *) iter->data)->action;
	if (input->rsc && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_casei)
	&& pcmk__str_eq(input->task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	migrating = TRUE;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	static bool
	should_lock_action(pe_action_t *action)
	{
	// Only actions taking place on resource's lock node are locked
	if ((action->rsc->lock_node == NULL) \|\| (action->node == NULL)
	\|\| (action->node->details != action->rsc->lock_node->details)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && action->task
	&& strcmp(action->task, RSC_STOP)) {
	return false;
	}

	return true;
	}

	static xmlNode *
	action2xml(pe_action_t * action, gboolean as_input, pe_working_set_t *data_set)
	{
	gboolean needs_node_info = TRUE;
	gboolean needs_maintenance_info = FALSE;
	xmlNode *action_xml = NULL;
	xmlNode *args_xml = NULL;
	#if ENABLE_VERSIONED_ATTRS
	pe_rsc_action_details_t *rsc_details = NULL;
	#endif

	if (action == NULL) {
	return NULL;
	}

	if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	/* All fences need node info; guest node fences are pseudo-events */
	action_xml = create_xml_node(NULL,
	pcmk_is_set(action->flags, pe_action_pseudo)?
	XML_GRAPH_TAG_PSEUDO_EVENT :
	XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	/* } else if(pcmk__str_eq(action->task, RSC_PROBED, pcmk__str_casei)) { */
	/* action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); */

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	if (pcmk__str_eq(action->task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
	needs_maintenance_info = TRUE;
	}
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = FALSE;

	} else {
	action_xml = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);

	#if ENABLE_VERSIONED_ATTRS
	rsc_details = pe_rsc_action_details(action);
	#endif
	}

	crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
	if (action->rsc != NULL && action->rsc->clone_name != NULL) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__str_eq(action->task, RSC_NOTIFY, pcmk__str_casei)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");

	CRM_CHECK(n_type != NULL, crm_err("No notify type value found for %s", action->uuid));
	CRM_CHECK(n_task != NULL,
	crm_err("No notify operation value found for %s", action->uuid));
	clone_key = pcmk__notify_key(action->rsc->clone_name,
	n_type, n_task);

	} else if(action->cancel_task) {
	clone_key = pcmk__op_key(action->rsc->clone_name,
	action->cancel_task, interval_ms);
	} else {
	clone_key = pcmk__op_key(action->rsc->clone_name,
	action->task, interval_ms);
	}

	CRM_CHECK(clone_key != NULL, crm_err("Could not generate a key for %s", action->uuid));
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY, action->uuid);
	free(clone_key);

	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if (needs_node_info && action->node != NULL) {
	pe_node_t *router_node = get_router_node(action);

	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
	crm_xml_add(action_xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
	if (router_node) {
	crm_xml_add(action_xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
	}

	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET), strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID), strdup(action->node->details->id));
	}

	/* No details if this action is only being listed in the inputs section */
	if (as_input) {
	return action_xml;
	}

	if (action->rsc && !pcmk_is_set(action->flags, pe_action_pseudo)) {
	int lpc = 0;
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER,
	XML_ATTR_TYPE
	};

	/* If a resource is locked to a node via shutdown-lock, mark its actions
	* so the controller can preserve the lock when the action completes.
	*/
	if (should_lock_action(action)) {
	crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	crm_element_name(action->rsc->xml));
	if (pcmk_is_set(action->rsc->flags, pe_rsc_orphan)
	&& action->rsc->clone_name) {
	/* Do not use the 'instance free' name here as that
	* might interfere with the instance we plan to keep.
	* Ie. if there are more than two named /anonymous/
	* instances on a given node, we need to make sure the
	* command goes to the right one.
	*
	* Keep this block, even when everyone is using
	* 'instance free' anonymous clone names - it means
	* we'll do the right thing if anyone toggles the
	* unique flag to 'off'
	*/
	crm_debug("Using orphan clone name %s instead of %s", action->rsc->id,
	action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pe_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka. %s)", xml_id, action->rsc->id,
	action->rsc->clone_name);

	/* ID is what we'd like client to use
	* ID_LONG is what they might know it as instead
	*
	* ID_LONG is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
	if (action->rsc->clone_name && !pcmk__str_eq(xml_id, action->rsc->clone_name, pcmk__str_casei)) {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
	}

	for (lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/* List any attributes in effect */
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
	crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	g_hash_table_foreach(action->extra, hash2field, args_xml);
	if (action->rsc != NULL && action->node) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node, data_set);

	/* REMOTE_CONTAINER_HACK: If this is a remote connection resource with
	* addr="#uname", pull the actual value from the parameters evaluated
	* without a node (which was put there earlier in stage8() when the
	* bundle's expand() method was called).
	*/
	const char *remote_addr = g_hash_table_lookup(params,
	XML_RSC_ATTR_REMOTE_RA_ADDR);

	if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
	GHashTable *base = pe_rsc_params(action->rsc, NULL, data_set);

	remote_addr = g_hash_table_lookup(base,
	XML_RSC_ATTR_REMOTE_RA_ADDR);
	if (remote_addr != NULL) {
	g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
	strdup(remote_addr));
	}
	}

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	#if ENABLE_VERSIONED_ATTRS
	{
	xmlNode *versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);

	pe_get_versioned_attributes(versioned_parameters, action->rsc,
	action->node, data_set);
	if (xml_has_children(versioned_parameters)) {
	add_node_copy(action_xml, versioned_parameters);
	}
	free_xml(versioned_parameters);
	}
	#endif

	} else if(action->rsc && action->rsc->variant <= pe_native) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL, data_set);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	#if ENABLE_VERSIONED_ATTRS
	if (xml_has_children(action->rsc->versioned_parameters)) {
	add_node_copy(action_xml, action->rsc->versioned_parameters);
	}
	#endif
	}

	#if ENABLE_VERSIONED_ATTRS
	if (rsc_details) {
	if (xml_has_children(rsc_details->versioned_parameters)) {
	add_node_copy(action_xml, rsc_details->versioned_parameters);
	}

	if (xml_has_children(rsc_details->versioned_meta)) {
	add_node_copy(action_xml, rsc_details->versioned_meta);
	}
	}
	#endif

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	const char *value = g_hash_table_lookup(action->rsc->meta, "external-ip");
	pe_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->append_meta(parent, args_xml);
	parent = parent->parent;
	}

	if(value) {
	hash2smartfield((gpointer)"pcmk_external_ip", (gpointer)value, (gpointer)args_xml);
	}

	if (action->node && /* make clang analyzer happy */
	pe__is_guest_node(action->node)) {
	pe_node_t *host = NULL;
	enum action_tasks task = text2task(action->task);

	if(task == action_notify \|\| task == action_notified) {
	const char *n_task = g_hash_table_lookup(action->meta, "notify_operation");
	task = text2task(n_task);
	}

	// Differentiate between up and down actions
	switch (task) {
	case stop_rsc:
	case stopped_rsc:
	case action_demote:
	case action_demoted:
	host = pe__current_node(action->node->details->remote_rsc->container);
	break;
	case start_rsc:
	case started_rsc:
	case monitor_rsc:
	case action_promote:
	case action_promoted:
	host = action->node->details->remote_rsc->container->allocated_to;
	break;
	default:
	break;
	}

	if(host) {
	hash2metafield((gpointer)XML_RSC_ATTR_TARGET,
	(gpointer)g_hash_table_lookup(action->rsc->meta, XML_RSC_ATTR_TARGET), (gpointer)args_xml);
	hash2metafield((gpointer) PCMK__ENV_PHYSICAL_HOST,
	(gpointer)host->details->uname,
	(gpointer)args_xml);
	}
	}

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei) && action->node) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action, data_set);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, data_set);
	}

	crm_log_xml_trace(action_xml, "dumped action");
	return action_xml;
	}

	static bool
	should_dump_action(pe_action_t *action)
	{
	CRM_CHECK(action != NULL, return false);

	if (pcmk_is_set(action->flags, pe_action_dumped)) {
	crm_trace("Action %s (%d) already dumped", action->uuid, action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)
	&& pcmk__str_eq(action->task, CRM_OP_PROBED, pcmk__str_casei)) {
	GList *lpc = NULL;

	/* This is a horrible but convenient hack
	*
	* It mimimizes the number of actions with unsatisfied inputs
	* (i.e. not included in the graph)
	*
	* This in turn, means we can be more concise when printing
	* aborted/incomplete graphs.
	*
	* It also makes it obvious which node is preventing
	* probe_complete from running (presumably because it is only
	* partially up)
	*
	* For these reasons we tolerate such perversions
	*/

	for (lpc = action->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t wrapper = (pe_action_wrapper_t ) lpc->data;

	if (!pcmk_is_set(wrapper->action->flags, pe_action_runnable)) {
	/* Only interested in runnable operations */
	} else if (!pcmk__str_eq(wrapper->action->task, RSC_START, pcmk__str_casei)) {
	/* Only interested in start operations */
	} else if (pcmk_is_set(wrapper->action->flags, pe_action_dumped)
	\|\| should_dump_action(wrapper->action)) {
	crm_trace("Action %s (%d) should be dumped: "
	"dependency of %s (%d)",
	action->uuid, action->id,
	wrapper->action->uuid, wrapper->action->id);
	return true;
	}
	}
	}

	if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_optional)
	&& !pcmk_is_set(action->flags, pe_action_print_always)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;

	// Monitors should be dumped even for unmanaged resources
	} else if (action->rsc && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
	&& !pcmk__str_eq(action->task, RSC_STATUS, pcmk__str_casei)) {

	const char *interval_ms_s = g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS);

	// Cancellation of recurring monitors should still be dumped
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	if (pcmk_is_set(action->flags, pe_action_pseudo) \|\|
	pcmk__strcase_any_of(action->task, CRM_OP_FENCE, CRM_OP_SHUTDOWN, NULL)) {
	/* skip the next checks */
	return true;
	}

	if (action->node == NULL) {
	pe_err("Skipping action %s (%d) "
	"because it was not allocated to a node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Unallocated action", action, false);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, action->node->details->uname);

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest node %s",
	action->uuid, action->id, action->node->details->uname);

	} else if (action->node->details->online == false) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for offline node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Action for offline node", action, FALSE);
	return false;
	#if 0
	/* but this would also affect resources that can be safely
	* migrated before a fencing op
	*/
	} else if (action->node->details->unclean == false) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for unclean node (bug?)",
	action->uuid, action->id);
	log_action(LOG_DEBUG, "Action for unclean node", action, false);
	return false;
	#endif
	}
	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pe_action_wrapper_t action_wrapper2 = (const pe_action_wrapper_t )a;
	const pe_action_wrapper_t action_wrapper1 = (const pe_action_wrapper_t )b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}

	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Check whether an ordering's flags can change an action
	*
	* \param[in] ordering Ordering to check
	*
	* \return true if ordering has flags that can change an action, false otherwise
	*/
	static bool
	ordering_can_change_actions(pe_action_wrapper_t *ordering)
	{
	return pcmk_any_flags_set(ordering->type, ~(pe_order_implies_first_printed
	\|pe_order_implies_then_printed
	\|pe_order_optional));
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	check_dump_input(pe_action_t action, pe_action_wrapper_t input)
	{
	if (input->state == pe_link_dumped) {
	return true;
	}

	if (input->type == pe_order_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& !ordering_can_change_actions(input)
	&& !pcmk__str_eq(input->action->uuid, CRM_OP_PROBED, pcmk__str_casei)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& pcmk_is_set(input->type, pe_order_one_or_more)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"one-or-more and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)
	&& pcmk_is_set(input->type, pe_order_stonith_stop)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"stonith stop but action is pseudo",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
	&& !pcmk_is_set(input->action->flags, pe_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"implies input migratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
	&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"only if input unmigratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((input->type == pe_order_optional)
	&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (input->type == pe_order_load) {
	pe_node_t *input_node = input->action->node;

	// load orderings are relevant only if actions are for same node

	if (action->rsc && pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)) {
	pe_node_t *allocated = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where it
	* has been allocated to, not where it will be executed.
	*/
	if ((input_node == NULL) \|\| (allocated == NULL)
	\|\| (input_node->details != allocated->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(allocated? allocated->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;
	}

	} else if ((input_node == NULL) \|\| (action->node == NULL)
	\|\| (input_node->details != action->node->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->type == pe_order_anti_colocation) {
	if (input->action->node && action->node
	&& (input->action->node->details != action->node->details)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	action->node->details->uname,
	input->action->node->details->uname);
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pe_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pe_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pe_action_print_always\|pe_action_dumped)
	&& !should_dump_action(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s 0x%.6x",
	action->uuid, action->id, action_type_str(input->action->flags),
	input->action->uuid, input->action->id,
	action_node_str(input->action),
	action_runnable_str(input->action->flags),
	action_optional_str(input->action->flags), input->type);
	return true;
	}

	static bool
	graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (0x%.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!check_dump_input(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pe__set_action_flags(input->action, pe_action_tracking);

	crm_trace("Checking inputs of action %s@%s input %s@%s (0x%.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (graph_has_loop(init_action, input->action,
	(pe_action_wrapper_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	goto done;
	}
	}

	done:
	pe__clear_action_flags(input->action, pe_action_tracking);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (0x%.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	bool
	pcmk__ordering_is_invalid(pe_action_t action, pe_action_wrapper_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pe_order_preserve)
	&& action->rsc && action->rsc->fillers
	&& input->action->rsc && input->action->node
	&& input->action->node->details->remote_rsc
	&& (input->action->node->details->remote_rsc->container == action->rsc)) {
	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->type == pe_order_load) && action->rsc
	&& pcmk__str_eq(action->task, RSC_MIGRATE, pcmk__str_casei)
	&& graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	// Remove duplicate inputs (regardless of flags)
	static void
	deduplicate_inputs(pe_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pe_action_wrapper_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) item->data;

	next = item->next;
	if (last_input && (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in] action Action to possibly add
	* \param[in] data_set Cluster working set
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pe_action_wrapper_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after stage7()
	* is complete, and nothing after this should rely on those type flags.
	* (For example, some code looks for type equal to some flag rather than
	* whether the flag is set, and some code looks for particular
	* combinations of flags -- such code must be done before stage8().)
	*/
	void
	graph_element_from_action(pe_action_t action, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	int synapse_priority = 0;
	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;
	xmlNode *xml_action = NULL;
	pe_action_wrapper_t *input = NULL;

	/* If we haven't already, de-duplicate inputs -- even if we won't be dumping
	* the action, so that crm_simulate dot graphs don't have duplicates.
	*/
	if (!pcmk_is_set(action->flags, pe_action_dedup)) {
	deduplicate_inputs(action);
	pe__set_action_flags(action, pe_action_dedup);
	}

	if (should_dump_action(action) == FALSE) {
	return;
	}

	pe__set_action_flags(action, pe_action_dumped);

	syn = create_xml_node(data_set->graph, "synapse");
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
	data_set->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}

	xml_action = action2xml(action, FALSE, data_set);
	add_node_nocopy(set, crm_element_name(xml_action), xml_action);

	for (lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	input = (pe_action_wrapper_t *) lpc->data;
	if (check_dump_input(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	xml_action = action2xml(input->action, TRUE, data_set);
	add_node_nocopy(input_xml, crm_element_name(xml_action), xml_action);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
	index bc111d68df..8a5778b6d6 100644
	--- a/lib/pacemaker/pcmk_sched_allocate.c
	+++ b/lib/pacemaker/pcmk_sched_allocate.c
	@@ -1,2902 +1,2900 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	extern bool pcmk__is_daemon;

	void set_alloc_actions(pe_working_set_t * data_set);
	extern void ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set);
	extern gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	static void apply_remote_node_ordering(pe_working_set_t *data_set);
	static enum remote_connection_state get_remote_node_state(pe_node_t *node);

	enum remote_connection_state {
	remote_state_unknown = 0,
	remote_state_alive = 1,
	remote_state_resting = 2,
	remote_state_failed = 3,
	remote_state_stopped = 4
	};

	static const char *
	state2text(enum remote_connection_state state)
	{
	switch (state) {
	case remote_state_unknown:
	return "unknown";
	case remote_state_alive:
	return "alive";
	case remote_state_resting:
	return "resting";
	case remote_state_failed:
	return "failed";
	case remote_state_stopped:
	return "stopped";
	}

	return "impossible";
	}

	resource_alloc_functions_t resource_class_alloc_functions[] = {
	{
	pcmk__native_merge_weights,
	pcmk__native_allocate,
	native_create_actions,
	native_create_probe,
	native_internal_constraints,
	native_rsc_colocation_lh,
	native_rsc_colocation_rh,
	native_rsc_location,
	native_action_flags,
	native_update_actions,
	native_expand,
	native_append_meta,
	},
	{
	pcmk__group_merge_weights,
	pcmk__group_allocate,
	group_create_actions,
	native_create_probe,
	group_internal_constraints,
	group_rsc_colocation_lh,
	group_rsc_colocation_rh,
	group_rsc_location,
	group_action_flags,
	group_update_actions,
	group_expand,
	group_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__clone_allocate,
	clone_create_actions,
	clone_create_probe,
	clone_internal_constraints,
	clone_rsc_colocation_lh,
	clone_rsc_colocation_rh,
	clone_rsc_location,
	clone_action_flags,
	pcmk__multi_update_actions,
	clone_expand,
	clone_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__bundle_allocate,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_rsc_colocation_lh,
	pcmk__bundle_rsc_colocation_rh,
	pcmk__bundle_rsc_location,
	pcmk__bundle_action_flags,
	pcmk__multi_update_actions,
	pcmk__bundle_expand,
	pcmk__bundle_append_meta,
	}
	};

	static gboolean
	check_rsc_parameters(pe_resource_t * rsc, pe_node_t * node, xmlNode * rsc_entry,
	gboolean active_here, pe_working_set_t * data_set)
	{
	int attr_lpc = 0;
	gboolean force_restart = FALSE;
	gboolean delete_resource = FALSE;
	gboolean changed = FALSE;

	const char *value = NULL;
	const char *old_value = NULL;

	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (; attr_lpc < PCMK__NELEM(attr_list); attr_lpc++) {
	value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
	old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
	if (value == old_value /* i.e. NULL */
	\|\| pcmk__str_eq(value, old_value, pcmk__str_none)) {
	continue;
	}

	changed = TRUE;
	trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
	if (active_here) {
	force_restart = TRUE;
	crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
	rsc->id, node->details->uname, attr_list[attr_lpc],
	crm_str(old_value), crm_str(value));
	}
	}
	if (force_restart) {
	/* make sure the restart happens */
	stop_action(rsc, node, FALSE);
	pe__set_resource_flags(rsc, pe_rsc_start_pending);
	delete_resource = TRUE;

	} else if (changed) {
	delete_resource = TRUE;
	}
	return delete_resource;
	}

	static void
	CancelXmlOp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * active_node,
	const char reason, pe_working_set_t data_set)
	{
	guint interval_ms = 0;
	pe_action_t *cancel = NULL;

	const char *task = NULL;
	const char *call_id = NULL;

	CRM_CHECK(xml_op != NULL, return);
	CRM_CHECK(active_node != NULL, return);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	crm_info("Action " PCMK__OP_FMT " on %s will be stopped: %s",
	rsc->id, task, interval_ms,
	active_node->details->uname, (reason? reason : "unknown"));

	cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
	custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
	}

	static gboolean
	check_action_definition(pe_resource_t * rsc, pe_node_t * active_node, xmlNode * xml_op,
	pe_working_set_t * data_set)
	{
	char *key = NULL;
	guint interval_ms = 0;
	const op_digest_cache_t *digest_data = NULL;
	gboolean did_change = FALSE;

	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_secure = NULL;

	CRM_CHECK(active_node != NULL, return FALSE);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms > 0) {
	xmlNode *op_match = NULL;

	/* we need to reconstruct the key because of the way we used to construct resource IDs */
	key = pcmk__op_key(rsc->id, task, interval_ms);

	pe_rsc_trace(rsc, "Checking parameters for %s", key);
	op_match = find_rsc_op_entry(rsc, key);

	if ((op_match == NULL)
	&& pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)) {
	CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
	free(key);
	return TRUE;

	} else if (op_match == NULL) {
	pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
	free(key);
	return TRUE;
	}
	free(key);
	key = NULL;
	}

	crm_trace("Testing " PCMK__OP_FMT " on %s",
	rsc->id, task, interval_ms, active_node->details->uname);
	if ((interval_ms == 0) && pcmk__str_eq(task, RSC_STATUS, pcmk__str_casei)) {
	/* Reload based on the start action not a probe */
	task = RSC_START;

	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_MIGRATED, pcmk__str_casei)) {
	/* Reload based on the start action not a migrate */
	task = RSC_START;
	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_PROMOTE, pcmk__str_casei)) {
	/* Reload based on the start action not a promote */
	task = RSC_START;
	}

	digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
	}

	if(digest_data->rc != RSC_DIGEST_MATCH
	&& digest_secure
	&& digest_data->digest_secure_calc
	&& strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
	if (!pcmk__is_daemon && data_set->priv != NULL) {
	pcmk__output_t *out = data_set->priv;
	out->info(out, "Only 'private' parameters to "
	PCMK__OP_FMT " on %s changed: %s", rsc->id, task,
	interval_ms, active_node->details->uname,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}

	} else if (digest_data->rc == RSC_DIGEST_RESTART) {
	/* Changes that force a restart */
	pe_action_t *required = NULL;

	did_change = TRUE;
	key = pcmk__op_key(rsc->id, task, interval_ms);
	crm_log_xml_info(digest_data->params_restart, "params:restart");
	required = custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);

	} else if ((digest_data->rc == RSC_DIGEST_ALL) \|\| (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
	// Changes that can potentially be handled by an agent reload
	const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	did_change = TRUE;
	trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
	crm_log_xml_info(digest_data->params_all, "params:reload");
	key = pcmk__op_key(rsc->id, task, interval_ms);

	if (interval_ms > 0) {
	pe_action_t *op = NULL;

	#if 0
	/* Always reload/restart the entire resource */
	ReloadRsc(rsc, active_node, data_set);
	#else
	/* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
	op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_reschedule);
	#endif

	} else if (digest_restart) {
	pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);

	/* Reload this resource */
	ReloadRsc(rsc, active_node, data_set);
	free(key);

	} else {
	pe_action_t *required = NULL;
	pe_rsc_trace(rsc, "Resource %s doesn't support agent reloads",
	rsc->id);

	/* Re-send the start/demote/promote op
	* Recurring ops will be detected independently
	*/
	required = custom_action(rsc, key, task, NULL, FALSE, TRUE,
	data_set);
	pe_action_set_reason(required, "resource definition change", true);
	}
	}

	return did_change;
	}

	/*!
	* \internal
	* \brief Do deferred action checks after allocation
	*
	* \param[in] data_set Working set for cluster
	*/
	static void
	check_params(pe_resource_t rsc, pe_node_t node, xmlNode *rsc_op,
	enum pe_check_parameters check, pe_working_set_t *data_set)
	{
	const char *reason = NULL;
	op_digest_cache_t *digest_data = NULL;

	switch (check) {
	case pe_check_active:
	if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	reason = "action definition changed";
	}
	break;

	case pe_check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s has no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}

	if (reason) {
	pe__clear_failcount(rsc, node, reason, data_set);
	}
	}

	static void
	check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;

	const char *task = NULL;

	xmlNode *rsc_op = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	CRM_CHECK(node != NULL, return);

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_resource_t *parent = uber_parent(rsc);
	if(parent == NULL
	\|\| pe_rsc_is_clone(parent) == FALSE
	\|\| pcmk_is_set(parent->flags, pe_rsc_unique)) {
	pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
	DeleteRsc(rsc, node, FALSE, data_set);
	} else {
	pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
	}
	return;

	} else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}
	pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
	rsc->id, node->details->uname);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);

	if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;
	guint interval_ms = 0;

	offset++;

	if (start_index < stop_index) {
	/* stopped */
	continue;
	} else if (offset < start_index) {
	/* action occurred prior to a start */
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0) &&
	(pcmk_is_set(rsc->flags, pe_rsc_maintenance) \|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);

	} else if ((interval_ms > 0) \|\| pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
	RSC_PROMOTE, RSC_MIGRATED, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc, data_set)) {
	/* We haven't allocated resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pe_check_active,
	data_set);

	} else if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	data_set);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	static GList *
	find_rsc_list(GList result, pe_resource_t rsc, const char *id, gboolean renamed_clones,
	gboolean partial, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	gboolean match = FALSE;

	if (id == NULL) {
	return NULL;
	}

	if (rsc == NULL) {
	if (data_set == NULL) {
	return NULL;
	}
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial,
	NULL);
	}
	return result;
	}

	if (partial) {
	if (strstr(rsc->id, id)) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
	match = TRUE;
	}

	} else {
	if (strcmp(rsc->id, id) == 0) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;
	}
	}

	if (match) {
	result = g_list_prepend(result, rsc);
	}

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
	}
	}

	return result;
	}

	static void
	check_actions(pe_working_set_t * data_set)
	{
	const char *id = NULL;
	pe_node_t *node = NULL;
	xmlNode *lrm_rscs = NULL;
	xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE,
	pcmk__str_none)) {
	id = crm_element_value(node_state, XML_ATTR_ID);
	lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);

	node = pe_find_node_id(data_set->nodes, id);

	if (node == NULL) {
	continue;

	/* Still need to check actions for a maintenance node to cancel existing monitor operations */
	} else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
	crm_trace("Skipping param check for %s: can't run resources",
	node->details->uname);
	continue;
	}

	crm_trace("Processing node %s", node->details->uname);
	if (node->details->online
	\|\| pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	xmlNode *rsc_entry = NULL;

	for (rsc_entry = pcmk__xe_first_child(lrm_rscs);
	rsc_entry != NULL;
	rsc_entry = pcmk__xe_next(rsc_entry)) {

	if (pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	if (xml_has_children(rsc_entry)) {
	GList *gIter = NULL;
	GList *result = NULL;
	const char *rsc_id = ID(rsc_entry);

	CRM_CHECK(rsc_id != NULL, return);

	result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
	for (gIter = result; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (rsc->variant != pe_native) {
	continue;
	}
	check_actions_for(rsc_entry, rsc, node, data_set);
	}
	g_list_free(result);
	}
	}
	}
	}
	}
	}
	}

	static void
	apply_placement_constraints(pe_working_set_t * data_set)
	{
	for (GList *gIter = data_set->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pe__location_t *cons = gIter->data;

	cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
	}
	}

	static gboolean
	failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
	{
	gboolean rc = FALSE;
	GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);

	if (list) {
	rc = TRUE;
	}
	g_list_free(list);
	return rc;
	}

	static void
	common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	common_apply_stickiness(child_rsc, node, data_set);
	}
	return;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)
	&& rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
	pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (current == NULL) {

	} else if ((match != NULL)
	\|\| pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	pe_resource_t *sticky_rsc = rsc;

	resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
	pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
	" (node=%s, weight=%d)", sticky_rsc->id,
	node->details->uname, rsc->stickiness);
	} else {
	GHashTableIter iter;
	pe_node_t *nIter = NULL;

	pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
	" and node %s is not explicitly allowed", rsc->id, node->details->uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
	crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
	}
	}
	}

	/* Check the migration threshold only if a failcount clear action
	* has not already been placed for this resource on the node.
	* There is no sense in potentially forcing the resource from this
	* node if the failcount is being reset anyway.
	*
	* @TODO A clear_failcount operation can be scheduled in stage4() via
	* check_actions_for(), or in stage5() via check_params(). This runs in
	* stage2(), so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't -- worst case, we stop or move the resource,
	* then move it back next transition.
	*/
	if (failcount_clear_action_exists(node, rsc) == FALSE) {
	pe_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, data_set, &failed)) {
	resource_location(failed, node, -INFINITY, "__fail_limit__",
	data_set);
	}
	}
	}

	void
	complex_set_cmds(pe_resource_t * rsc)
	{
	GList *gIter = rsc->children;

	rsc->cmds = &resource_class_alloc_functions[rsc->variant];

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(child_rsc);
	}
	}

	void
	set_alloc_actions(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(rsc);
	}
	}

	static void
	calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
	{
	const char key = (const char )gKey;
	const char value = (const char )gValue;
	int system_health = (int )user_data;

	if (!gKey \|\| !gValue \|\| !user_data) {
	return;
	}

	if (pcmk__starts_with(key, "#health")) {
	int score;

	/* Convert the value into an integer */
	score = char2score(value);

	/* Add it to the running total */
	system_health = pe__add_scores(score, system_health);
	}
	}

	static gboolean
	apply_system_health(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
	int base_health = 0;

	if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches \| pcmk__str_casei)) {
	/* Prevent any accidental health -> score translation */
	pcmk__score_red = 0;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;
	return TRUE;

	} else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red are
	* weighted away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red or yellow
	* are forced away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = -INFINITY;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
	/* Same as the above, but use the r/y/g scores provided by the user
	* Defaults are provided by the pe_prefs table
	* Also, custom health "base score" can be used
	*/
	base_health = char2score(pe_pref(data_set->config_hash,
	"node-health-base"));

	} else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {

	/* Requires the admin to configure the rsc_location constaints for
	* processing the stored health scores
	*/
	/* TODO: Check for the existence of appropriate node health constraints */
	return TRUE;

	} else {
	crm_err("Unknown node health strategy: %s", health_strategy);
	return FALSE;
	}

	crm_info("Applying automated node health strategy: %s", health_strategy);

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int system_health = base_health;
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Search through the node hash table for system health entries. */
	g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);

	crm_info(" Node %s has an combined system health of %d",
	node->details->uname, system_health);

	/* If the health is non-zero, then create a new rsc2node so that the
	* weight will be added later on.
	*/
	if (system_health != 0) {

	GList *gIter2 = data_set->resources;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
	}
	}
	}

	return TRUE;
	}

	gboolean
	stage0(pe_working_set_t * data_set)
	{
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	if (data_set->input == NULL) {
	return FALSE;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
	crm_trace("Calculating status");
	cluster_status(data_set);
	}

	set_alloc_actions(data_set);
	apply_system_health(data_set);
	unpack_constraints(cib_constraints, data_set);

	return TRUE;
	}

	/*
	* Check nodes for resources started outside of the LRM
	*/
	gboolean
	probe_resources(pe_working_set_t * data_set)
	{
	pe_action_t *probe_node_complete = NULL;

	for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);

	if (node->details->online == FALSE) {

	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& (get_remote_node_state(node) == remote_state_failed)) {

	pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) {
	continue;

	} else if (node->details->rsc_discovery_enabled == FALSE) {
	/* resource discovery is disabled for this node */
	continue;
	}

	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pe_action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
	CRM_OP_REPROBE, node, FALSE, TRUE, data_set);

	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	continue;
	}

	for (GList *gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
	}
	}
	return TRUE;
	}

	static void
	rsc_discover_filter(pe_resource_t rsc, pe_node_t node)
	{
	pe_resource_t *top = uber_parent(rsc);
	pe_node_t *match;

	if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
	return;
	}

	g_list_foreach(rsc->children, (GFunc) rsc_discover_filter, node);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match && match->rsc_discover_mode != pe_discover_exclusive) {
	match->weight = -INFINITY;
	}
	}

	static time_t
	shutdown_time(pe_node_t node, pe_working_set_t data_set)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return result? result : get_effective_time(data_set);
	}

	static void
	apply_shutdown_lock(pe_resource_t rsc, pe_working_set_t data_set)
	{
	const char *class;

	// Only primitives and (uncloned) groups may be locked
	if (rsc->variant == pe_group) {
	g_list_foreach(rsc->children, (GFunc) apply_shutdown_lock, data_set);
	} else if (rsc->variant != pe_native) {
	return;
	}

	// Fence devices and remote connections can't be locked
	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc, data_set)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node, data_set);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, node->details->uname);
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node, data_set);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (data_set->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, rsc->lock_node->details->uname,
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, data_set);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, rsc->lock_node->details->uname);
	}

	// If resource is locked to one node, ban it from all other nodes
	for (GList *item = data_set->nodes; item != NULL; item = item->next) {
	pe_node_t *node = item->data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
	}
	}
	}

	/*
	* \internal
	* \brief Stage 2 of cluster status: apply node-specific criteria
	*
	* Count known nodes, and apply location constraints, stickiness, and exclusive
	* resource discovery.
	*/
	gboolean
	stage2(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	g_list_foreach(data_set->resources, (GFunc) apply_shutdown_lock, data_set);
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
	// @COMPAT API backward compatibility
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	data_set->max_valid_nodes++;
	}
	}
	}

	apply_placement_constraints(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	GList *gIter2 = NULL;
	pe_node_t node = (pe_node_t ) gIter->data;

	gIter2 = data_set->resources;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	common_apply_stickiness(rsc, node, data_set);
	rsc_discover_filter(rsc, node);
	}
	}

	return TRUE;
	}

	/*
	* Create internal resource constraints before allocation
	*/
	gboolean
	stage3(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->internal_constraints(rsc, data_set);
	}

	return TRUE;
	}

	/*
	* Check for orphaned or redefined actions
	*/
	gboolean
	stage4(pe_working_set_t * data_set)
	{
	check_actions(data_set);
	return TRUE;
	}

	static void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	static gint
	sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
	{
	int rc = 0;
	int r1_weight = -INFINITY;
	int r2_weight = -INFINITY;

	const char *reason = "existence";

	GList nodes = (GList ) data;
	const pe_resource_t *resource1 = a;
	const pe_resource_t *resource2 = b;

	pe_node_t *r1_node = NULL;
	pe_node_t *r2_node = NULL;
	GList *gIter = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;

	reason = "priority";
	r1_weight = resource1->priority;
	r2_weight = resource2->priority;

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	r1_nodes = pcmk__native_merge_weights(convert_const_pointer(resource1),
	resource1->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);

	r2_nodes = pcmk__native_merge_weights(convert_const_pointer(resource2),
	resource2->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	/* Current location score */
	reason = "current location";
	r1_weight = -INFINITY;
	r2_weight = -INFINITY;

	if (resource1->running_on) {
	r1_node = pe__current_node(resource1);
	r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
	if (r1_node != NULL) {
	r1_weight = r1_node->weight;
	}
	}
	if (resource2->running_on) {
	r2_node = pe__current_node(resource2);
	r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
	if (r2_node != NULL) {
	r2_weight = r2_node->weight;
	}
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "score";
	for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	r1_node = NULL;
	r2_node = NULL;

	r1_weight = -INFINITY;
	if (r1_nodes) {
	r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
	}
	if (r1_node) {
	r1_weight = r1_node->weight;
	}

	r2_weight = -INFINITY;
	if (r2_nodes) {
	r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
	}
	if (r2_node) {
	r2_weight = r2_node->weight;
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
	resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
	rc < 0 ? '>' : rc > 0 ? '<' : '=',
	resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);

	if (r1_nodes) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes) {
	g_hash_table_destroy(r2_nodes);
	}

	return rc;
	}

	static void
	allocate_resources(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Allocate remote connection resources first (which will also allocate
	* any colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == FALSE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
	rsc->id);
	rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
	}
	}

	/* now do the rest of the resources */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == TRUE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating %s resource '%s'",
	crm_element_name(rsc->xml), rsc->id);
	rsc->cmds->allocate(rsc, NULL, data_set);
	}
	}

	/* We always use pe_order_preserve with these convenience functions to exempt
	* internally generated constraints from the prohibition of user constraints
	* involving remote connection resources.
	*
	* The start ordering additionally uses pe_order_runnable_left so that the
	* specified action is not runnable if the start is not runnable.
	*/

	static inline void
	order_start_then_action(pe_resource_t lh_rsc, pe_action_t rh_action,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_rsc && rh_action && data_set) {
	custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
	rh_action->rsc, NULL, rh_action,
	pe_order_preserve \| pe_order_runnable_left \| extra,
	data_set);
	}
	}

	static inline void
	order_action_then_stop(pe_action_t lh_action, pe_resource_t rh_rsc,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_action && rh_rsc && data_set) {
	custom_action_order(lh_action->rsc, NULL, lh_action,
	rh_rsc, stop_key(rh_rsc), NULL,
	pe_order_preserve \| extra, data_set);
	}
	}

	// Clear fail counts for orphaned rsc on all online nodes
	static void
	cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node->details->online
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	pe_action_t *clear_op = NULL;

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	data_set);

	/* We can't use order_action_then_stop() here because its
	* pe_order_preserve breaks things
	*/
	custom_action_order(clear_op->rsc, NULL, clear_op,
	rsc, stop_key(rsc), NULL,
	pe_order_optional, data_set);
	}
	}
	}

	gboolean
	stage5(pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *gIter = NULL;

	if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
	GList *nodes = g_list_copy(data_set->nodes);

	nodes = sort_nodes_by_weight(nodes, NULL, data_set);
	data_set->resources =
	g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);

	g_list_free(nodes);
	}

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Original");
	}
	}

	crm_trace("Allocating services");
	/* Take (next) highest resource, assign it and create its actions */

	allocate_resources(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	out->message(out, "node-capacity", node, "Remaining");
	}
	}

	// Process deferred action checks
	pe__foreach_param_check(data_set, check_params);
	pe__free_param_checks(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Calculating needed probes");
	/* This code probably needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=100:

	With probes:

	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
	ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	36s
	ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph

	Without probes:

	ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
	*/

	probe_resources(data_set);
	}

	crm_trace("Handle orphans");
	if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* There's no need to recurse into rsc->children because those
	* should just be unallocated clone instances.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	cleanup_orphans(rsc, data_set);
	}
	}
	}

	crm_trace("Creating actions");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->create_actions(rsc, data_set);
	}

	crm_trace("Creating done");
	return TRUE;
	}

	static gboolean
	is_managed(const pe_resource_t * rsc)
	{
	GList *gIter = rsc->children;

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (is_managed(child_rsc)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	any_managed_resources(pe_working_set_t * data_set)
	{

	GList *gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (is_managed(rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*
	* Create dependencies for stonith and shutdown operations
	*/
	gboolean
	stage6(pe_working_set_t * data_set)
	{
	pe_action_t *dc_down = NULL;
	pe_action_t *stonith_op = NULL;
	gboolean integrity_lost = FALSE;
	gboolean need_stonith = TRUE;
	GList *gIter;
	GList *stonith_ops = NULL;
	GList *shutdown_ops = NULL;

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we will mark the node as dirty.
	*
	* A nice side effect of doing them early is that apply_*_ordering() can be
	* simpler because pe_fence_node() has already done some of the work.
	*/
	crm_trace("Creating remote ordering constraints");
	apply_remote_node_ordering(data_set);

	crm_trace("Processing fencing and shutdown cases");
	if (any_managed_resources(data_set) == FALSE) {
	crm_notice("Delaying fencing operations until there are resources to manage");
	need_stonith = FALSE;
	}

	/* Check each node for stonith/shutdown */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && need_stonith
	&& pe_can_fence(data_set, node)) {
	pcmk__fence_guest(node, data_set);
	}
	continue;
	}

	stonith_op = NULL;

	if (node->details->unclean
	&& need_stonith && pe_can_fence(data_set, node)) {

	stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
	pe_warn("Scheduling Node %s for STONITH", node->details->uname);

	pcmk__order_vs_fence(stonith_op, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being fenced
	dc_down = stonith_op;

	} else {

	if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
	&& (stonith_ops != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pe_action_t *) stonith_ops->data,
	stonith_op, pe_order_optional);
	}

	// Remember all non-DC fencing actions in a separate list
	stonith_ops = g_list_prepend(stonith_ops, stonith_op);
	}

	} else if (node->details->online && node->details->shutdown &&
	/* TODO define what a shutdown op means for a remote node.
	* For now we do not send shutdown operations for remote nodes, but
	* if we can come up with a good use for this in the future, we will. */
	pe__is_guest_or_remote_node(node) == FALSE) {

	pe_action_t *down_op = sched_shutdown_op(node, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being shut down
	dc_down = down_op;
	} else {
	// Remember non-DC shutdowns for later ordering
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if (node->details->unclean && stonith_op == NULL) {
	integrity_lost = TRUE;
	pe_warn("Node %s is unclean!", node->details->uname);
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
	pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	crm_notice("Cannot fence unclean nodes until quorum is"
	" attained (or no-quorum-policy is set to ignore)");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
	pe_action_t node_stop = (pe_action_t ) gIter->data;

	crm_debug("Ordering shutdown on %s before %s on DC %s",
	node_stop->node->details->uname,
	dc_down->task, dc_down->node->details->uname);

	order_actions(node_stop, dc_down, pe_order_optional);
	}
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
	order_actions((pe_action_t *) gIter->data, dc_down,
	pe_order_optional);
	}
	} else if (stonith_ops) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pe_action_t *) stonith_ops->data, dc_down,
	pe_order_optional);
	}
	}
	g_list_free(stonith_ops);
	g_list_free(shutdown_ops);
	return TRUE;
	}

	/*
	* Determine the sets of independent actions and the correct order for the
	* actions in each set.
	*
	* Mark dependencies of un-runnable actions un-runnable
	*
	*/
	static GList *
	find_actions_by_task(GList actions, pe_resource_t rsc, const char *original_key)
	{
	GList *list = NULL;

	list = find_actions(actions, original_key, NULL);
	if (list == NULL) {
	/* we're potentially searching a child of the original resource */
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(actions, key, NULL);

	} else {
	crm_err("search key: %s", original_key);
	}

	free(key);
	free(task);
	}

	return list;
	}

	static void
	rsc_order_then(pe_action_t lh_action, pe_resource_t rsc,
	pe__ordering_t *order)
	{
	GList *gIter = NULL;
	GList *rh_actions = NULL;
	pe_action_t *rh_action = NULL;
	enum pe_ordering type;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(order != NULL, return);

	type = order->type;
	rh_action = order->rh_action;
	crm_trace("Applying ordering constraint %d (then: %s)", order->id, rsc->id);

	if (rh_action != NULL) {
	rh_actions = g_list_prepend(NULL, rh_action);

	} else if (rsc != NULL) {
	rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
	}

	if (rh_actions == NULL) {
	pe_rsc_trace(rsc,
	"Ignoring constraint %d: then (%s for %s) not found",
	order->id, order->rh_action_task, rsc->id);
	return;
	}

	if ((lh_action != NULL) && (lh_action->rsc == rsc)
	&& pcmk_is_set(lh_action->flags, pe_action_dangle)) {

	pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
	order->rh_action_task);
	pe__clear_order_flags(type, pe_order_implies_then);
	}

	gIter = rh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) gIter->data;

	if (lh_action) {
	order_actions(lh_action, rh_action_iter, type);

	} else if (type & pe_order_implies_then) {
	pe__clear_action_flags(rh_action_iter, pe_action_runnable);
	crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
	} else {
	crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
	}
	}

	g_list_free(rh_actions);
	}

	static void
	rsc_order_first(pe_resource_t lh_rsc, pe__ordering_t order,
	pe_working_set_t *data_set)
	{
	GList *lh_actions = NULL;
	pe_action_t *lh_action = order->lh_action;
	pe_resource_t *rh_rsc = order->rh_rsc;

	CRM_ASSERT(lh_rsc != NULL);
	pe_rsc_trace(lh_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, lh_rsc->id);

	if (lh_action != NULL) {
	lh_actions = g_list_prepend(NULL, lh_action);

	} else {
	lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
	}

	if ((lh_actions == NULL) && (lh_rsc == rh_rsc)) {
	pe_rsc_trace(lh_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, lh_rsc->id);

	} else if (lh_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(lh_rsc->id, op_type, interval_ms);

	if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, lh_rsc->id);

	} else if ((lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_UNPROMOTED)
	&& pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, lh_rsc->id);

	} else {
	pe_rsc_trace(lh_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->lh_action_task, lh_rsc->id, order->id);
	lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
	lh_actions = g_list_prepend(NULL, lh_action);
	}

	free(op_type);
	}

	if (rh_rsc == NULL) {
	if (order->rh_action == NULL) {
	pe_rsc_trace(lh_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	rh_rsc = order->rh_action->rsc;
	}
	for (GList *gIter = lh_actions; gIter != NULL; gIter = gIter->next) {
	lh_action = (pe_action_t *) gIter->data;

	if (rh_rsc == NULL) {
	order_actions(lh_action, order->rh_action, order->type);

	} else {
	rsc_order_then(lh_action, rh_rsc, order);
	}
	}

	g_list_free(lh_actions);
	}

	-extern void update_colo_start_chain(pe_action_t *action,
	- pe_working_set_t *data_set);
	-
	static int
	is_recurring_action(pe_action_t *action)
	{
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return 0;
	}
	return (interval_ms > 0);
	}

	static void
	apply_container_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	/* VMs are also classified as containers for these purposes... in
	* that they both involve a 'thing' running on a real or remote
	* cluster node.
	*
	* This allows us to be smarter about the type and extent of
	* recovery actions required in various scenarios
	*/
	pe_resource_t *remote_rsc = NULL;
	pe_resource_t *container = NULL;
	enum action_tasks task = text2task(action->task);

	CRM_ASSERT(action->rsc);
	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	container = remote_rsc->container;
	CRM_ASSERT(container);

	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	pe_fence_node(data_set, action->node, "container failed", FALSE);
	}

	crm_trace("Order %s action %s relative to %s%s for %s%s",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id,
	pcmk_is_set(container->flags, pe_rsc_failed)? "failed " : "",
	container->id);

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	/* Force resource recovery if the container is recovered */
	order_start_then_action(container, action, pe_order_implies_then,
	data_set);

	/* Wait for the connection resource to be up too */
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	break;

	case stop_rsc:
	case action_demote:
	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	/* When the container representing a guest node fails, any stop
	* or demote actions for resources running on the guest node
	* are implied by the container stopping. This is similar to
	* how fencing operations work for cluster nodes and remote
	* nodes.
	*/
	} else {
	/* Ensure the operation happens before the connection is brought
	* down.
	*
	* If we really wanted to, we could order these after the
	* connection start, IFF the container's current role was
	* stopped (otherwise we re-introduce an ordering loop when the
	* connection is restarting).
	*/
	order_action_then_stop(action, remote_rsc, pe_order_none,
	data_set);
	}
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	if(task != no_action) {
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);
	}
	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	break;
	}
	}

	static enum remote_connection_state
	get_remote_node_state(pe_node_t *node)
	{
	pe_resource_t *remote_rsc = NULL;
	pe_node_t *cluster_node = NULL;

	CRM_ASSERT(node);

	remote_rsc = node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	cluster_node = pe__current_node(remote_rsc);

	/* If the cluster node the remote connection resource resides on
	* is unclean or went offline, we can't process any operations
	* on that remote node until after it starts elsewhere.
	*/
	if(remote_rsc->next_role == RSC_ROLE_STOPPED \|\| remote_rsc->allocated_to == NULL) {
	/* The connection resource is not going to run anywhere */

	if (cluster_node && cluster_node->details->unclean) {
	/* The remote connection is failed because its resource is on a
	* failed node and can't be recovered elsewhere, so we must fence.
	*/
	return remote_state_failed;
	}

	if (!pcmk_is_set(remote_rsc->flags, pe_rsc_failed)) {
	/* Connection resource is cleanly stopped */
	return remote_state_stopped;
	}

	/* Connection resource is failed */

	if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
	&& remote_rsc->remote_reconnect_ms
	&& node->details->remote_was_fenced
	&& !pe__shutdown_requested(node)) {

	/* We won't know whether the connection is recoverable until the
	* reconnect interval expires and we reattempt connection.
	*/
	return remote_state_unknown;
	}

	/* The remote connection is in a failed state. If there are any
	* resources known to be active on it (stop) or in an unknown state
	* (probe), we must assume the worst and fence it.
	*/
	return remote_state_failed;

	} else if (cluster_node == NULL) {
	/* Connection is recoverable but not currently running anywhere, see if we can recover it first */
	return remote_state_unknown;

	} else if(cluster_node->details->unclean == TRUE
	\|\| cluster_node->details->online == FALSE) {
	/* Connection is running on a dead node, see if we can recover it first */
	return remote_state_resting;

	} else if (pcmk__list_of_multiple(remote_rsc->running_on)
	&& remote_rsc->partial_migration_source
	&& remote_rsc->partial_migration_target) {
	/* We're in the middle of migrating a connection resource,
	* wait until after the resource migrates before performing
	* any actions.
	*/
	return remote_state_resting;

	}
	return remote_state_alive;
	}

	/*!
	* \internal
	* \brief Order actions on remote node relative to actions for the connection
	*/
	static void
	apply_remote_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	pe_resource_t *remote_rsc = NULL;
	enum action_tasks task = text2task(action->task);
	enum remote_connection_state state = get_remote_node_state(action->node);

	enum pe_ordering order_opts = pe_order_none;

	if (action->rsc == NULL) {
	return;
	}

	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	crm_trace("Order %s action %s relative to %s%s (state: %s)",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id, state2text(state));

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	order_opts = pe_order_none;

	if (state == remote_state_failed) {
	/* Force recovery, by making this action required */
	pe__set_order_flags(order_opts, pe_order_implies_then);
	}

	/* Ensure connection is up before running this action */
	order_start_then_action(remote_rsc, action, order_opts, data_set);
	break;

	case stop_rsc:
	if(state == remote_state_alive) {
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else if(state == remote_state_failed) {
	/* The resource is active on the node, but since we don't have a
	* valid connection, the only way to stop the resource is by
	* fencing the node. There is no need to order the stop relative
	* to the remote connection, since the stop will become implied
	* by the fencing.
	*/
	pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE);

	} else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
	/* State must be remote_state_unknown or remote_state_stopped.
	* Since the connection is not coming back up in this
	* transition, stop this resource first.
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else {
	/* The connection is going to be started somewhere else, so
	* stop this resource after that completes.
	*/
	order_start_then_action(remote_rsc, action, pe_order_none, data_set);
	}
	break;

	case action_demote:
	/* Only order this demote relative to the connection start if the
	* connection isn't being torn down. Otherwise, the demote would be
	* blocked because the connection start would not be allowed.
	*/
	if(state == remote_state_resting \|\| state == remote_state_unknown) {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	} /* Otherwise we can rely on the stop ordering */
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);

	} else {
	pe_node_t *cluster_node = pe__current_node(remote_rsc);

	if(task == monitor_rsc && state == remote_state_failed) {
	/* We would only be here if we do not know the
	* state of the resource on the remote node.
	* Since we have no way to find out, it is
	* necessary to fence the node.
	*/
	pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE);
	}

	if(cluster_node && state == remote_state_stopped) {
	/* The connection is currently up, but is going
	* down permanently.
	*
	* Make sure we check services are actually
	* stopped _before_ we let the connection get
	* closed
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_runnable_left, data_set);

	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	}
	break;
	}
	}

	static void
	apply_remote_node_ordering(pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	return;
	}

	for (GList *gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_resource_t *remote = NULL;

	// We are only interested in resource actions
	if (action->rsc == NULL) {
	continue;
	}

	/* Special case: If we are clearing the failcount of an actual
	* remote connection resource, then make sure this happens before
	* any start of the resource in this transition.
	*/
	if (action->rsc->is_remote_node &&
	pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {

	custom_action_order(action->rsc,
	NULL,
	action,
	action->rsc,
	pcmk__op_key(action->rsc->id, RSC_START, 0),
	NULL,
	pe_order_optional,
	data_set);

	continue;
	}

	// We are only interested in actions allocated to a node
	if (action->node == NULL) {
	continue;
	}

	if (!pe__is_guest_or_remote_node(action->node)) {
	continue;
	}

	/* We are only interested in real actions.
	*
	* @TODO This is probably wrong; pseudo-actions might be converted to
	* real actions and vice versa later in update_actions() at the end of
	* stage7().
	*/
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	continue;
	}

	remote = action->node->details->remote_rsc;
	if (remote == NULL) {
	// Orphaned
	continue;
	}

	/* Another special case: if a resource is moving to a Pacemaker Remote
	* node, order the stop on the original node after any start of the
	* remote connection. This ensures that if the connection fails to
	* start, we leave the resource running on the original node.
	*/
	if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	for (GList *item = action->rsc->actions; item != NULL;
	item = item->next) {
	pe_action_t *rsc_action = item->data;

	if ((rsc_action->node->details != action->node->details)
	&& pcmk__str_eq(rsc_action->task, RSC_STOP, pcmk__str_casei)) {
	custom_action_order(remote, start_key(remote), NULL,
	action->rsc, NULL, rsc_action,
	pe_order_optional, data_set);
	}
	}
	}

	/* The action occurs across a remote connection, so create
	* ordering constraints that guarantee the action occurs while the node
	* is active (after start, before stop ... things like that).
	*
	* This is somewhat brittle in that we need to make sure the results of
	* this ordering are compatible with the result of get_router_node().
	* It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
	* of this logic rather than action2xml().
	*/
	if (remote->container) {
	crm_trace("Container ordering for %s", action->uuid);
	apply_container_ordering(action, data_set);

	} else {
	crm_trace("Remote ordering for %s", action->uuid);
	apply_remote_ordering(action, data_set);
	}
	}
	}

	static gboolean
	order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
	{
	/* No need to probe the resource on the node that is being
	* unfenced. Otherwise it might introduce transition loop
	* since probe will be performed after the node is
	* unfenced.
	*/
	if (pcmk__str_eq(rh_action->task, CRM_OP_FENCE, pcmk__str_casei)
	&& probe->node && rh_action->node
	&& probe->node->details == rh_action->node->details) {
	const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");

	if (pcmk__str_eq(op, "on", pcmk__str_casei)) {
	return TRUE;
	}
	}

	// Shutdown waits for probe to complete only if it's on the same node
	if ((pcmk__str_eq(rh_action->task, CRM_OP_SHUTDOWN, pcmk__str_casei))
	&& probe->node && rh_action->node
	&& probe->node->details != rh_action->node->details) {
	return TRUE;
	}
	return FALSE;
	}

	static void
	order_first_probes_imply_stops(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	enum pe_ordering order_type = pe_order_optional;

	pe_resource_t *lh_rsc = order->lh_rsc;
	pe_resource_t *rh_rsc = order->rh_rsc;
	pe_action_t *lh_action = order->lh_action;
	pe_action_t *rh_action = order->rh_action;
	const char *lh_action_task = order->lh_action_task;
	const char *rh_action_task = order->rh_action_task;

	GList *probes = NULL;
	GList *rh_actions = NULL;

	GList *pIter = NULL;

	if (lh_rsc == NULL) {
	continue;

	} else if (rh_rsc && lh_rsc == rh_rsc) {
	continue;
	}

	if (lh_action == NULL && lh_action_task == NULL) {
	continue;
	}

	if (rh_action == NULL && rh_action_task == NULL) {
	continue;
	}

	/* Technically probe is expected to return "not running", which could be
	* the alternative of stop action if the status of the resource is
	* unknown yet.
	*/
	if (lh_action && !pcmk__str_eq(lh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (lh_action == NULL
	&& lh_action_task
	&& !pcmk__ends_with(lh_action_task, "_" RSC_STOP "_0")) {
	continue;
	}

	/* Do not probe the resource inside of a stopping container. Otherwise
	* it might introduce transition loop since probe will be performed
	* after the container starts again.
	*/
	if (rh_rsc && lh_rsc->container == rh_rsc) {
	if (rh_action && pcmk__str_eq(rh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (rh_action == NULL && rh_action_task
	&& pcmk__ends_with(rh_action_task,"_" RSC_STOP "_0")) {
	continue;
	}
	}

	if (order->type == pe_order_none) {
	continue;
	}

	// Preserve the order options for future filtering
	if (pcmk_is_set(order->type, pe_order_apply_first_non_migratable)) {
	pe__set_order_flags(order_type,
	pe_order_apply_first_non_migratable);
	}

	if (pcmk_is_set(order->type, pe_order_same_node)) {
	pe__set_order_flags(order_type, pe_order_same_node);
	}

	// Keep the order types for future filtering
	if (order->type == pe_order_anti_colocation
	\|\| order->type == pe_order_load) {
	order_type = order->type;
	}

	probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
	if (probes == NULL) {
	continue;
	}

	if (rh_action) {
	rh_actions = g_list_prepend(rh_actions, rh_action);

	} else if (rh_rsc && rh_action_task) {
	rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
	}

	if (rh_actions == NULL) {
	g_list_free(probes);
	continue;
	}

	crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
	" (id=%d, type=%.6x)",
	lh_action ? lh_action->uuid : lh_action_task,
	rh_action ? rh_action->uuid : rh_action_task,
	order->id, order->type);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;
	GList *rIter = NULL;

	for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) rIter->data;

	if (order_first_probe_unneeded(probe, rh_action_iter)) {
	continue;
	}
	order_actions(probe, rh_action_iter, order_type);
	}
	}

	g_list_free(rh_actions);
	g_list_free(probes);
	}
	}

	static void
	order_first_probe_then_restart_repromote(pe_action_t * probe,
	pe_action_t * after,
	pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	bool interleave = FALSE;
	pe_resource_t *compatible_rsc = NULL;

	if (probe == NULL
	\|\| probe->rsc == NULL
	\|\| probe->rsc->variant != pe_native) {
	return;
	}

	if (after == NULL
	// Avoid running into any possible loop
	\|\| pcmk_is_set(after->flags, pe_action_tracking)) {
	return;
	}

	if (!pcmk__str_eq(probe->task, RSC_STATUS, pcmk__str_casei)) {
	return;
	}

	pe__set_action_flags(after, pe_action_tracking);

	crm_trace("Processing based on %s %s -> %s %s",
	probe->uuid,
	probe->node ? probe->node->details->uname: "",
	after->uuid,
	after->node ? after->node->details->uname : "");

	if (after->rsc
	/* Better not build a dependency directly with a clone/group.
	* We are going to proceed through the ordering chain and build
	* dependencies with its children.
	*/
	&& after->rsc->variant == pe_native
	&& probe->rsc != after->rsc) {

	GList *then_actions = NULL;
	enum pe_ordering probe_order_type = pe_order_optional;

	if (pcmk__str_eq(after->task, RSC_START, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, RSC_PROMOTE, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_DEMOTE, FALSE);
	}

	for (gIter = then_actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t then = (pe_action_t ) gIter->data;

	// Skip any pseudo action which for example is implied by fencing
	if (pcmk_is_set(then->flags, pe_action_pseudo)) {
	continue;
	}

	order_actions(probe, then, probe_order_type);
	}
	g_list_free(then_actions);
	}

	if (after->rsc
	&& after->rsc->variant > pe_group) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);

	if (interleave) {
	/* For an interleaved clone, we should build a dependency only
	* with the relevant clone child.
	*/
	compatible_rsc = find_compatible_child(probe->rsc,
	after->rsc,
	RSC_ROLE_UNKNOWN,
	FALSE, data_set);
	}
	}

	for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) gIter->data;
	/* pe_order_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pe_order_implies_then_on_node though. It's now only
	* used for unfencing case, which tends to introduce transition
	* loops...
	*/

	if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pe_order_implies_first_printed \| pe_order_runnable_left
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if (after->rsc == NULL
	\|\| after->rsc->variant < pe_group
	\|\| probe->rsc->parent == after->rsc
	\|\| after_wrapper->action->rsc == NULL
	\|\| after_wrapper->action->rsc->variant > pe_group
	\|\| after->rsc != after_wrapper->action->rsc->parent) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if (after->rsc->variant > pe_group
	&& interleave == TRUE
	&& (compatible_rsc == NULL
	\|\| compatible_rsc != after_wrapper->action->rsc)) {
	continue;
	}
	}

	crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)",
	after->uuid,
	after->node ? after->node->details->uname: "",
	after_wrapper->action->uuid,
	after_wrapper->action->node ? after_wrapper->action->node->details->uname : "",
	after_wrapper->type);

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	}
	}

	static void clear_actions_tracking_flag(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (pcmk_is_set(action->flags, pe_action_tracking)) {
	pe__clear_action_flags(action, pe_action_tracking);
	}
	}
	}

	static void
	order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	GList *probes = NULL;

	g_list_foreach(rsc->children, (GFunc) order_first_rsc_probes, data_set);

	if (rsc->variant != pe_native) {
	return;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (gIter = probes; gIter != NULL; gIter= gIter->next) {
	pe_action_t probe = (pe_action_t ) gIter->data;
	GList *aIter = NULL;

	for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) aIter->data;

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	clear_actions_tracking_flag(data_set);
	}
	}

	g_list_free(probes);
	}

	static void
	order_first_probes(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	order_first_rsc_probes(rsc, data_set);
	}

	order_first_probes_imply_stops(data_set);
	}

	static void
	order_then_probes(pe_working_set_t * data_set)
	{
	#if 0
	GList *gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* Given "A then B", we would prefer to wait for A to be
	* started before probing B.
	*
	* If A was a filesystem on which the binaries and data for B
	* lived, it would have been useful if the author of B's agent
	* could assume that A is running before B.monitor will be
	* called.
	*
	* However we can't _only_ probe once A is running, otherwise
	* we'd not detect the state of B if A could not be started
	* for some reason.
	*
	* In practice however, we cannot even do an opportunistic
	* version of this because B may be moving:
	*
	* B.probe -> B.start
	* B.probe -> B.stop
	* B.stop -> B.start
	* A.stop -> A.start
	* A.start -> B.probe
	*
	* So far so good, but if we add the result of this code:
	*
	* B.stop -> A.stop
	*
	* Then we get a loop:
	*
	* B.probe -> B.stop -> A.stop -> A.start -> B.probe
	*
	* We could kill the 'B.probe -> B.stop' dependency, but that
	* could mean stopping B "too" soon, because B.start must wait
	* for the probes to complete.
	*
	* Another option is to allow it only if A is a non-unique
	* clone with clone-max == node-max (since we'll never be
	* moving it). However, we could still be stopping one
	* instance at the same time as starting another.

	* The complexity of checking for allowed conditions combined
	* with the ever narrowing usecase suggests that this code
	* should remain disabled until someone gets smarter.
	*/
	pe_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if(start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (actions = start->actions_before; actions != NULL; actions = actions->next) {
	pe_action_wrapper_t before = (pe_action_wrapper_t ) actions->data;

	GList *pIter = NULL;
	pe_action_t *first = before->action;
	pe_resource_t *first_rsc = first->rsc;

	if(first->required_runnable_before) {
	GList *clone_actions = NULL;
	for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
	before = (pe_action_wrapper_t *) clone_actions->data;

	crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);

	CRM_ASSERT(before->action->rsc);
	first_rsc = before->action->rsc;
	break;
	}

	} else if(!pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if(first_rsc == NULL) {
	continue;

	} else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pe_order_optional);
	}
	}
	}
	#endif
	}

	static void
	order_probes(pe_working_set_t * data_set)
	{
	order_first_probes(data_set);
	order_then_probes(data_set);
	}

	gboolean
	stage7(pe_working_set_t * data_set)
	{
	pcmk__output_t *prev_out = data_set->priv;
	pcmk__output_t *out = NULL;
	GList *gIter = NULL;

	crm_trace("Applying ordering constraints");

	/* Don't ask me why, but apparently they need to be processed in
	* the order they were created in... go figure
	*
	* Also g_list_append() has horrendous performance characteristics
	* So we need to use g_list_prepend() and then reverse the list here
	*/
	data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	pe_resource_t *rsc = order->lh_rsc;

	if (rsc != NULL) {
	rsc_order_first(rsc, order, data_set);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	rsc_order_then(order->lh_action, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->lh_action, order->rh_action, order->type);
	}
	}

	- g_list_foreach(data_set->actions, (GFunc) update_colo_start_chain, data_set);
	+ g_list_foreach(data_set->actions, (GFunc) pcmk__block_colocated_starts,
	+ data_set);

	crm_trace("Ordering probes");
	order_probes(data_set);

	crm_trace("Updating %d actions", g_list_length(data_set->actions));
	g_list_foreach(data_set->actions, (GFunc) update_action, data_set);

	// Check for invalid orderings
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (pcmk__ordering_is_invalid(action, input)) {
	input->type = pe_order_none;
	}
	}
	}

	/* stage7 only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	out = pcmk__new_logger();
	if (out == NULL) {
	return FALSE;
	}

	pcmk__output_set_log_level(out, LOG_NOTICE);
	data_set->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	LogNodeActions(data_set);

	g_list_foreach(data_set->resources, (GFunc) LogActions, data_set);

	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	data_set->priv = prev_out;
	return TRUE;
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*
	* Create a dependency graph to send to the transitioner (via the controller)
	*/
	gboolean
	stage8(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d.", transition_id);

	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(data_set->config_hash, "cluster-delay");
	crm_xml_add(data_set->graph, "cluster-delay", value);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	crm_xml_add(data_set->graph, "stonith-timeout", value);

	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
	crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(data_set->graph, "failed-start-offset", "1");
	}

	value = pe_pref(data_set->config_hash, "batch-limit");
	crm_xml_add(data_set->graph, "batch-limit", value);

	crm_xml_add_int(data_set->graph, "transition_id", transition_id);

	value = pe_pref(data_set->config_hash, "migration-limit");
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(data_set->graph, "migration-limit", value);
	}

	if (data_set->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) data_set->recheck_by);
	crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	gIter = data_set->resources;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
	rsc->cmds->expand(rsc, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created resource-driven action list");

	/* pseudo action to distribute list of nodes with maintenance state update */
	add_maintenance_update(data_set);

	/* catch any non-resource specific actions */
	crm_trace("processing non-resource actions");

	gIter = data_set->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc
	&& action->node
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pe_action_optional\|pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)
	) {
	/* Eventually we should just ignore the 'fence' case
	* But for now it's the best way to detect (in CTS) when
	* CIB resource updates are being lost
	*/
	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	\|\| data_set->no_quorum_policy == no_quorum_ignore) {
	crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
	action->node->details->unclean ? "fence" : "shut down",
	action->node->details->uname, action->rsc->id,
	pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
	pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
	action->uuid);
	}
	}

	graph_element_from_action(action, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created generic action list");
	crm_trace("Created transition graph %d.", transition_id);

	return TRUE;
	}

	void
	LogNodeActions(pe_working_set_t * data_set)
	{
	pcmk__output_t *out = data_set->priv;
	GList *gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc != NULL) {
	continue;
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;
	}

	if (pe__is_guest_node(action->node)) {
	node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
	} else if(action->node) {
	node_name = crm_strdup_printf("%s", action->node->details->uname);
	}


	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	task = strdup("Shutdown");
	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");
	task = crm_strdup_printf("Fence (%s)", op);
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_bundle.c b/lib/pacemaker/pcmk_sched_bundle.c
	index 660cc085f9..c8edaca34a 100644
	--- a/lib/pacemaker/pcmk_sched_bundle.c
	+++ b/lib/pacemaker/pcmk_sched_bundle.c
	@@ -1,1064 +1,1066 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	+#include "libpacemaker_private.h"
	+
	#define PE__VARIANT_BUNDLE 1
	#include <lib/pengine/variant.h>

	static bool
	is_bundle_node(pe__bundle_variant_data_t data, pe_node_t node)
	{
	for (GList *gIter = data->replicas; gIter != NULL; gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (node->details == replica->node->details) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
	void distribute_children(pe_resource_t rsc, GList children, GList *nodes,
	int max, int per_host_max, pe_working_set_t * data_set);

	static GList *
	get_container_list(pe_resource_t *rsc)
	{
	GList *containers = NULL;

	if (rsc->variant == pe_container) {
	pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, rsc);
	for (GList *gIter = data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	containers = g_list_append(containers, replica->container);
	}
	}
	return containers;
	}

	static inline GList *
	get_containers_or_children(pe_resource_t *rsc)
	{
	return (rsc->variant == pe_container)?
	get_container_list(rsc) : rsc->children;
	}

	pe_node_t *
	pcmk__bundle_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	GList *containers = NULL;
	GList *nodes = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return NULL);

	get_bundle_variant_data(bundle_data, rsc);

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	containers = get_container_list(rsc);

	pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, data_set);

	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = sort_nodes_by_weight(nodes, NULL, data_set);
	containers = g_list_sort_with_data(containers, sort_clone_instance, data_set);
	distribute_children(rsc, containers, nodes, bundle_data->nreplicas,
	bundle_data->nreplicas_per_host, data_set);
	g_list_free(nodes);
	g_list_free(containers);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;
	pe_node_t *container_host = NULL;

	CRM_ASSERT(replica);
	if (replica->ip) {
	pe_rsc_trace(rsc, "Allocating bundle %s IP %s",
	rsc->id, replica->ip->id);
	replica->ip->cmds->allocate(replica->ip, prefer, data_set);
	}

	container_host = replica->container->allocated_to;
	if (replica->remote && pe__is_guest_or_remote_node(container_host)) {
	/* We need 'nested' connection resources to be on the same
	* host because pacemaker-remoted only supports a single
	* active connection
	*/
	pcmk__new_colocation("child-remote-with-docker-remote", NULL,
	INFINITY, replica->remote,
	container_host->details->remote_rsc, NULL,
	NULL, true, data_set);
	}

	if (replica->remote) {
	pe_rsc_trace(rsc, "Allocating bundle %s connection %s",
	rsc->id, replica->remote->id);
	replica->remote->cmds->allocate(replica->remote, prefer,
	data_set);
	}

	// Explicitly allocate replicas' children before bundle child
	if (replica->child) {
	pe_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, replica->child->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	if (node->details != replica->node->details) {
	node->weight = -INFINITY;
	} else if (!pcmk__threshold_reached(replica->child, node,
	data_set, NULL)) {
	node->weight = INFINITY;
	}
	}

	pe__set_resource_flags(replica->child->parent, pe_rsc_allocating);
	pe_rsc_trace(rsc, "Allocating bundle %s replica child %s",
	rsc->id, replica->child->id);
	replica->child->cmds->allocate(replica->child, replica->node,
	data_set);
	pe__clear_resource_flags(replica->child->parent,
	pe_rsc_allocating);
	}
	}

	if (bundle_data->child) {
	pe_node_t *node = NULL;
	GHashTableIter iter;
	g_hash_table_iter_init(&iter, bundle_data->child->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	if (is_bundle_node(bundle_data, node)) {
	node->weight = 0;
	} else {
	node->weight = -INFINITY;
	}
	}
	pe_rsc_trace(rsc, "Allocating bundle %s child %s",
	rsc->id, bundle_data->child->id);
	bundle_data->child->cmds->allocate(bundle_data->child, prefer, data_set);
	}

	pe__clear_resource_flags(rsc, pe_rsc_allocating\|pe_rsc_provisional);
	return NULL;
	}


	void
	pcmk__bundle_create_actions(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe_action_t *action = NULL;
	GList *containers = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return);

	containers = get_container_list(rsc);
	get_bundle_variant_data(bundle_data, rsc);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (replica->ip) {
	replica->ip->cmds->create_actions(replica->ip, data_set);
	}
	if (replica->container) {
	replica->container->cmds->create_actions(replica->container,
	data_set);
	}
	if (replica->remote) {
	replica->remote->cmds->create_actions(replica->remote, data_set);
	}
	}

	clone_create_pseudo_actions(rsc, containers, NULL, NULL, data_set);

	if (bundle_data->child) {
	bundle_data->child->cmds->create_actions(bundle_data->child, data_set);

	if (pcmk_is_set(bundle_data->child->flags, pe_rsc_promotable)) {
	/* promote */
	create_pseudo_resource_op(rsc, RSC_PROMOTE, TRUE, TRUE, data_set);
	action = create_pseudo_resource_op(rsc, RSC_PROMOTED, TRUE, TRUE, data_set);
	action->priority = INFINITY;

	/* demote */
	create_pseudo_resource_op(rsc, RSC_DEMOTE, TRUE, TRUE, data_set);
	action = create_pseudo_resource_op(rsc, RSC_DEMOTED, TRUE, TRUE, data_set);
	action->priority = INFINITY;
	}
	}

	g_list_free(containers);
	}

	void
	pcmk__bundle_internal_constraints(pe_resource_t *rsc,
	pe_working_set_t *data_set)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);

	if (bundle_data->child) {
	new_rsc_order(rsc, RSC_START, bundle_data->child, RSC_START,
	pe_order_implies_first_printed, data_set);
	new_rsc_order(rsc, RSC_STOP, bundle_data->child, RSC_STOP,
	pe_order_implies_first_printed, data_set);

	if (bundle_data->child->children) {
	new_rsc_order(bundle_data->child, RSC_STARTED, rsc, RSC_STARTED,
	pe_order_implies_then_printed, data_set);
	new_rsc_order(bundle_data->child, RSC_STOPPED, rsc, RSC_STOPPED,
	pe_order_implies_then_printed, data_set);
	} else {
	new_rsc_order(bundle_data->child, RSC_START, rsc, RSC_STARTED,
	pe_order_implies_then_printed, data_set);
	new_rsc_order(bundle_data->child, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_implies_then_printed, data_set);
	}
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	CRM_ASSERT(replica->container);

	replica->container->cmds->internal_constraints(replica->container,
	data_set);

	order_start_start(rsc, replica->container,
	pe_order_runnable_left\|pe_order_implies_first_printed);

	if (replica->child) {
	order_stop_stop(rsc, replica->child,
	pe_order_implies_first_printed);
	}
	order_stop_stop(rsc, replica->container,
	pe_order_implies_first_printed);
	new_rsc_order(replica->container, RSC_START, rsc, RSC_STARTED,
	pe_order_implies_then_printed, data_set);
	new_rsc_order(replica->container, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_implies_then_printed, data_set);

	if (replica->ip) {
	replica->ip->cmds->internal_constraints(replica->ip, data_set);

	// Start ip then container
	new_rsc_order(replica->ip, RSC_START, replica->container, RSC_START,
	pe_order_runnable_left\|pe_order_preserve, data_set);
	new_rsc_order(replica->container, RSC_STOP, replica->ip, RSC_STOP,
	pe_order_implies_first\|pe_order_preserve, data_set);

	pcmk__new_colocation("ip-with-docker", NULL, INFINITY, replica->ip,
	replica->container, NULL, NULL, true,
	data_set);
	}

	if (replica->remote) {
	/* This handles ordering and colocating remote relative to container
	* (via "resource-with-container"). Since IP is also ordered and
	* colocated relative to the container, we don't need to do anything
	* explicit here with IP.
	*/
	replica->remote->cmds->internal_constraints(replica->remote,
	data_set);
	}

	if (replica->child) {
	CRM_ASSERT(replica->remote);

	// "Start remote then child" is implicit in scheduler's remote logic
	}

	}

	if (bundle_data->child) {
	bundle_data->child->cmds->internal_constraints(bundle_data->child, data_set);
	if (pcmk_is_set(bundle_data->child->flags, pe_rsc_promotable)) {
	promote_demote_constraints(rsc, data_set);

	/* child demoted before global demoted */
	new_rsc_order(bundle_data->child, RSC_DEMOTED, rsc, RSC_DEMOTED,
	pe_order_implies_then_printed, data_set);

	/* global demote before child demote */
	new_rsc_order(rsc, RSC_DEMOTE, bundle_data->child, RSC_DEMOTE,
	pe_order_implies_first_printed, data_set);

	/* child promoted before global promoted */
	new_rsc_order(bundle_data->child, RSC_PROMOTED, rsc, RSC_PROMOTED,
	pe_order_implies_then_printed, data_set);

	/* global promote before child promote */
	new_rsc_order(rsc, RSC_PROMOTE, bundle_data->child, RSC_PROMOTE,
	pe_order_implies_first_printed, data_set);
	}

	} else {
	// int type = pe_order_optional \| pe_order_implies_then \| pe_order_restart;
	// custom_action_order(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	// rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set);
	}
	}

	static pe_resource_t *
	compatible_replica_for_node(pe_resource_t rsc_lh, pe_node_t candidate,
	pe_resource_t *rsc, enum rsc_role_e filter,
	gboolean current)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(candidate != NULL, return NULL);
	get_bundle_variant_data(bundle_data, rsc);

	crm_trace("Looking for compatible child from %s for %s on %s",
	rsc_lh->id, rsc->id, candidate->details->uname);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (is_child_compatible(replica->container, candidate, filter, current)) {
	crm_trace("Pairing %s with %s on %s",
	rsc_lh->id, replica->container->id,
	candidate->details->uname);
	return replica->container;
	}
	}

	crm_trace("Can't pair %s with %s", rsc_lh->id, rsc->id);
	return NULL;
	}

	static pe_resource_t *
	compatible_replica(pe_resource_t rsc_lh, pe_resource_t rsc,
	enum rsc_role_e filter, gboolean current,
	pe_working_set_t *data_set)
	{
	GList *scratch = NULL;
	pe_resource_t *pair = NULL;
	pe_node_t *active_node_lh = NULL;

	active_node_lh = rsc_lh->fns->location(rsc_lh, NULL, current);
	if (active_node_lh) {
	return compatible_replica_for_node(rsc_lh, active_node_lh, rsc, filter,
	current);
	}

	scratch = g_hash_table_get_values(rsc_lh->allowed_nodes);
	scratch = sort_nodes_by_weight(scratch, NULL, data_set);

	for (GList *gIter = scratch; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	pair = compatible_replica_for_node(rsc_lh, node, rsc, filter, current);
	if (pair) {
	goto done;
	}
	}

	pe_rsc_debug(rsc, "Can't pair %s with %s", rsc_lh->id, (rsc? rsc->id : "none"));
	done:
	g_list_free(scratch);
	return pair;
	}

	void
	pcmk__bundle_rsc_colocation_lh(pe_resource_t rsc, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	/* -- Never called --
	*
	* Instead we add the colocation constraints to the child and call from there
	*/
	CRM_ASSERT(FALSE);
	}

	int copies_per_node(pe_resource_t * rsc)
	{
	/* Strictly speaking, there should be a 'copies_per_node' addition
	* to the resource function table and each case would be a
	* function. However that would be serious overkill to return an
	* int. In fact, it seems to me that both function tables
	* could/should be replaced by resources.{c,h} full of
	* rsc_{some_operation} functions containing a switch as below
	* which calls out to functions named {variant}_{some_operation}
	* as needed.
	*/
	switch(rsc->variant) {
	case pe_unknown:
	return 0;
	case pe_native:
	case pe_group:
	return 1;
	case pe_clone:
	{
	const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);

	if (max_clones_node == NULL) {
	return 1;

	} else {
	int max_i;

	pcmk__scan_min_int(max_clones_node, &max_i, 0);
	return max_i;
	}
	}
	case pe_container:
	{
	pe__bundle_variant_data_t *data = NULL;
	get_bundle_variant_data(data, rsc);
	return data->nreplicas_per_host;
	}
	}
	return 0;
	}

	void
	pcmk__bundle_rsc_colocation_rh(pe_resource_t rsc_lh, pe_resource_t rsc,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *allocated_rhs = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(constraint != NULL, return);
	CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
	CRM_CHECK(rsc != NULL, pe_err("rsc was NULL for %s", constraint->id); return);
	CRM_ASSERT(rsc_lh->variant == pe_native);

	if (pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	pe_rsc_trace(rsc, "%s is still provisional", rsc->id);
	return;

	} else if(constraint->rsc_lh->variant > pe_group) {
	pe_resource_t *rh_child = compatible_replica(rsc_lh, rsc,
	RSC_ROLE_UNKNOWN, FALSE,
	data_set);

	if (rh_child) {
	pe_rsc_debug(rsc, "Pairing %s with %s", rsc_lh->id, rh_child->id);
	rsc_lh->cmds->rsc_colocation_lh(rsc_lh, rh_child, constraint,
	data_set);

	} else if (constraint->score >= INFINITY) {
	crm_notice("Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
	assign_node(rsc_lh, NULL, TRUE);

	} else {
	pe_rsc_debug(rsc, "Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
	}

	return;
	}

	get_bundle_variant_data(bundle_data, rsc);
	pe_rsc_trace(rsc, "Processing constraint %s: %s -> %s %d",
	constraint->id, rsc_lh->id, rsc->id, constraint->score);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (constraint->score < INFINITY) {
	replica->container->cmds->rsc_colocation_rh(rsc_lh,
	replica->container,
	constraint, data_set);

	} else {
	pe_node_t *chosen = replica->container->fns->location(replica->container,
	NULL, FALSE);

	if ((chosen == NULL)
	\|\| is_set_recursive(replica->container, pe_rsc_block, TRUE)) {
	continue;
	}
	if ((constraint->role_rh >= RSC_ROLE_PROMOTED)
	&& (replica->child == NULL)) {
	continue;
	}
	if ((constraint->role_rh >= RSC_ROLE_PROMOTED)
	&& (replica->child->next_role < RSC_ROLE_PROMOTED)) {
	continue;
	}

	pe_rsc_trace(rsc, "Allowing %s: %s %d", constraint->id, chosen->details->uname, chosen->weight);
	allocated_rhs = g_list_prepend(allocated_rhs, chosen);
	}
	}

	if (constraint->score >= INFINITY) {
	node_list_exclude(rsc_lh->allowed_nodes, allocated_rhs, FALSE);
	}
	g_list_free(allocated_rhs);
	}

	enum pe_action_flags
	pcmk__bundle_action_flags(pe_action_t action, pe_node_t node)
	{
	GList *containers = NULL;
	enum pe_action_flags flags = 0;
	pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, action->rsc);
	if(data->child) {
	enum action_tasks task = get_complex_task(data->child, action->task, TRUE);
	switch(task) {
	case no_action:
	case action_notify:
	case action_notified:
	case action_promote:
	case action_promoted:
	case action_demote:
	case action_demoted:
	return summary_action_flags(action, data->child->children, node);
	default:
	break;
	}
	}

	containers = get_container_list(action->rsc);
	flags = summary_action_flags(action, containers, node);
	g_list_free(containers);
	return flags;
	}

	pe_resource_t *
	find_compatible_child_by_node(pe_resource_t * local_child, pe_node_t * local_node, pe_resource_t * rsc,
	enum rsc_role_e filter, gboolean current)
	{
	GList *gIter = NULL;
	GList *children = NULL;

	if (local_node == NULL) {
	crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
	return NULL;
	}

	crm_trace("Looking for compatible child from %s for %s on %s",
	local_child->id, rsc->id, local_node->details->uname);

	children = get_containers_or_children(rsc);
	for (gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if(is_child_compatible(child_rsc, local_node, filter, current)) {
	crm_trace("Pairing %s with %s on %s",
	local_child->id, child_rsc->id, local_node->details->uname);
	return child_rsc;
	}
	}

	crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
	if(children != rsc->children) {
	g_list_free(children);
	}
	return NULL;
	}

	static pe__bundle_replica_t *
	replica_for_container(pe_resource_t rsc, pe_resource_t container,
	pe_node_t *node)
	{
	if (rsc->variant == pe_container) {
	pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, rsc);
	for (GList *gIter = data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (replica->child
	&& (container == replica->container)
	&& (node->details == replica->node->details)) {
	return replica;
	}
	}
	}
	return NULL;
	}

	static enum pe_graph_flags
	multi_update_interleave_actions(pe_action_t first, pe_action_t then,
	pe_node_t *node, enum pe_action_flags flags,
	enum pe_action_flags filter,
	enum pe_ordering type,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	GList *children = NULL;
	gboolean current = FALSE;
	enum pe_graph_flags changed = pe_graph_none;

	/* Fix this - lazy */
	if (pcmk__ends_with(first->uuid, "_stopped_0")
	\|\| pcmk__ends_with(first->uuid, "_demoted_0")) {
	current = TRUE;
	}

	children = get_containers_or_children(then->rsc);
	for (gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t *then_child = gIter->data;
	pe_resource_t *first_child = find_compatible_child(then_child,
	first->rsc,
	RSC_ROLE_UNKNOWN,
	current, data_set);
	if (first_child == NULL && current) {
	crm_trace("Ignore");

	} else if (first_child == NULL) {
	crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, then->uuid);

	/* Me no like this hack - but what else can we do?
	*
	* If there is no-one active or about to be active
	* on the same node as then_child, then they must
	* not be allowed to start
	*/
	if (type & (pe_order_runnable_left \| pe_order_implies_then) /* Mandatory */ ) {
	pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
	if(assign_node(then_child, NULL, TRUE)) {
	pe__set_graph_flags(changed, first, pe_graph_updated_then);
	}
	}

	} else {
	pe_action_t *first_action = NULL;
	pe_action_t *then_action = NULL;

	enum action_tasks task = clone_child_action(first);
	const char *first_task = task2text(task);

	pe__bundle_replica_t *first_replica = NULL;
	pe__bundle_replica_t *then_replica = NULL;

	first_replica = replica_for_container(first->rsc, first_child,
	node);
	if (strstr(first->task, "stop") && first_replica && first_replica->child) {
	/* Except for 'stopped' we should be looking at the
	* in-container resource, actions for the child will
	* happen later and are therefor more likely to align
	* with the user's intent.
	*/
	first_action = find_first_action(first_replica->child->actions,
	NULL, task2text(task), node);
	} else {
	first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
	}

	then_replica = replica_for_container(then->rsc, then_child, node);
	if (strstr(then->task, "mote")
	&& then_replica && then_replica->child) {
	/* Promote/demote actions will never be found for the
	* container resource, look in the child instead
	*
	* Alternatively treat:
	* 'XXXX then promote YYYY' as 'XXXX then start container for YYYY', and
	* 'demote XXXX then stop YYYY' as 'stop container for XXXX then stop YYYY'
	*/
	then_action = find_first_action(then_replica->child->actions,
	NULL, then->task, node);
	} else {
	then_action = find_first_action(then_child->actions, NULL, then->task, node);
	}

	if (first_action == NULL) {
	if (!pcmk_is_set(first_child->flags, pe_rsc_orphan)
	&& !pcmk__str_any_of(first_task, RSC_STOP, RSC_DEMOTE, NULL)) {
	crm_err("Internal error: No action found for %s in %s (first)",
	first_task, first_child->id);

	} else {
	crm_trace("No action found for %s in %s%s (first)",
	first_task, first_child->id,
	pcmk_is_set(first_child->flags, pe_rsc_orphan)? " (ORPHAN)" : "");
	}
	continue;
	}

	/* We're only interested if 'then' is neither stopping nor being demoted */
	if (then_action == NULL) {
	if (!pcmk_is_set(then_child->flags, pe_rsc_orphan)
	&& !pcmk__str_any_of(then->task, RSC_STOP, RSC_DEMOTE, NULL)) {
	crm_err("Internal error: No action found for %s in %s (then)",
	then->task, then_child->id);

	} else {
	crm_trace("No action found for %s in %s%s (then)",
	then->task, then_child->id,
	pcmk_is_set(then_child->flags, pe_rsc_orphan)? " (ORPHAN)" : "");
	}
	continue;
	}

	if (order_actions(first_action, then_action, type)) {
	crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x",
	first_action->uuid,
	pcmk_is_set(first_action->flags, pe_action_optional),
	then_action->uuid,
	pcmk_is_set(then_action->flags, pe_action_optional),
	type);
	pe__set_graph_flags(changed, first,
	pe_graph_updated_first\|pe_graph_updated_then);
	}
	if(first_action && then_action) {
	changed \|= then_child->cmds->update_actions(first_action,
	then_action, node,
	first_child->cmds->action_flags(first_action, node),
	filter, type, data_set);
	} else {
	crm_err("Nothing found either for %s (%p) or %s (%p) %s",
	first_child->id, first_action,
	then_child->id, then_action, task2text(task));
	}
	}
	}

	if(children != then->rsc->children) {
	g_list_free(children);
	}
	return changed;
	}

	static bool
	can_interleave_actions(pe_action_t first, pe_action_t then)
	{
	bool interleave = FALSE;
	pe_resource_t *rsc = NULL;
	const char *interleave_s = NULL;

	if(first->rsc == NULL \|\| then->rsc == NULL) {
	crm_trace("Not interleaving %s with %s (both must be resources)", first->uuid, then->uuid);
	return FALSE;
	} else if(first->rsc == then->rsc) {
	crm_trace("Not interleaving %s with %s (must belong to different resources)", first->uuid, then->uuid);
	return FALSE;
	} else if(first->rsc->variant < pe_clone \|\| then->rsc->variant < pe_clone) {
	crm_trace("Not interleaving %s with %s (both sides must be clones or bundles)", first->uuid, then->uuid);
	return FALSE;
	}

	if (pcmk__ends_with(then->uuid, "_stop_0")
	\|\| pcmk__ends_with(then->uuid, "_demote_0")) {
	rsc = first->rsc;
	} else {
	rsc = then->rsc;
	}

	interleave_s = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERLEAVE);
	interleave = crm_is_true(interleave_s);
	crm_trace("Interleave %s -> %s: %s (based on %s)",
	first->uuid, then->uuid, interleave ? "yes" : "no", rsc->id);

	return interleave;
	}

	enum pe_graph_flags
	pcmk__multi_update_actions(pe_action_t first, pe_action_t then,
	pe_node_t *node, enum pe_action_flags flags,
	enum pe_action_flags filter, enum pe_ordering type,
	pe_working_set_t *data_set)
	{
	enum pe_graph_flags changed = pe_graph_none;

	crm_trace("%s -> %s", first->uuid, then->uuid);

	if(can_interleave_actions(first, then)) {
	changed = multi_update_interleave_actions(first, then, node, flags,
	filter, type, data_set);

	} else if(then->rsc) {
	GList *gIter = NULL;
	GList *children = NULL;

	// Handle the 'primitive' ordering case
	changed \|= native_update_actions(first, then, node, flags, filter,
	type, data_set);

	// Now any children (or containers in the case of a bundle)
	children = get_containers_or_children(then->rsc);
	for (gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t then_child = (pe_resource_t ) gIter->data;
	enum pe_graph_flags then_child_changed = pe_graph_none;
	pe_action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node);

	if (then_child_action) {
	enum pe_action_flags then_child_flags = then_child->cmds->action_flags(then_child_action, node);

	if (pcmk_is_set(then_child_flags, pe_action_runnable)) {
	then_child_changed \|= then_child->cmds->update_actions(first,
	then_child_action, node, flags, filter, type, data_set);
	}
	changed \|= then_child_changed;
	if (then_child_changed & pe_graph_updated_then) {
	for (GList *lpc = then_child_action->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t next = (pe_action_wrapper_t ) lpc->data;
	update_action(next->action, data_set);
	}
	}
	}
	}

	if(children != then->rsc->children) {
	g_list_free(children);
	}
	}
	return changed;
	}

	void
	pcmk__bundle_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	get_bundle_variant_data(bundle_data, rsc);

	native_rsc_location(rsc, constraint);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (replica->container) {
	replica->container->cmds->rsc_location(replica->container,
	constraint);
	}
	if (replica->ip) {
	replica->ip->cmds->rsc_location(replica->ip, constraint);
	}
	}

	if (bundle_data->child
	&& ((constraint->role_filter == RSC_ROLE_UNPROMOTED)
	\|\| (constraint->role_filter == RSC_ROLE_PROMOTED))) {
	bundle_data->child->cmds->rsc_location(bundle_data->child, constraint);
	bundle_data->child->rsc_location = g_list_prepend(bundle_data->child->rsc_location,
	constraint);
	}
	}

	void
	pcmk__bundle_expand(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);

	if (bundle_data->child) {
	bundle_data->child->cmds->expand(bundle_data->child, data_set);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (replica->remote && replica->container
	&& pe__bundle_needs_remote_name(replica->remote, data_set)) {

	/* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that
	* run pacemaker-remoted inside, without needing a separate IP for
	* the container. This is done by configuring the inner remote's
	* connection host as the magic string "#uname", then
	* replacing it with the underlying host when needed.
	*/
	xmlNode *nvpair = get_xpath_object("//nvpair[@name='" XML_RSC_ATTR_REMOTE_RA_ADDR "']",
	replica->remote->xml, LOG_ERR);
	const char *calculated_addr = NULL;

	// Replace the value in replica->remote->xml (if appropriate)
	calculated_addr = pe__add_bundle_remote_name(replica->remote,
	data_set,
	nvpair, "value");
	if (calculated_addr) {
	/* Since this is for the bundle as a resource, and not any
	* particular action, replace the value in the default
	* parameters (not evaluated for node). action2xml() will grab
	* it from there to replace it in node-evaluated parameters.
	*/
	GHashTable *params = pe_rsc_params(replica->remote,
	NULL, data_set);

	crm_trace("Set address for bundle connection %s to bundle host %s",
	replica->remote->id, calculated_addr);
	g_hash_table_replace(params,
	strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
	strdup(calculated_addr));
	} else {
	/* The only way to get here is if the remote connection is
	* neither currently running nor scheduled to run. That means we
	* won't be doing any operations that require addr (only start
	* requires it; we additionally use it to compare digests when
	* unpacking status, promote, and migrate_from history, but
	* that's already happened by this point).
	*/
	crm_info("Unable to determine address for bundle %s remote connection",
	rsc->id);
	}
	}
	if (replica->ip) {
	replica->ip->cmds->expand(replica->ip, data_set);
	}
	if (replica->container) {
	replica->container->cmds->expand(replica->container, data_set);
	}
	if (replica->remote) {
	replica->remote->cmds->expand(replica->remote, data_set);
	}
	}
	}

	gboolean
	pcmk__bundle_create_probe(pe_resource_t rsc, pe_node_t node,
	pe_action_t *complete, gboolean force,
	pe_working_set_t * data_set)
	{
	bool any_created = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return FALSE);

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (replica->ip) {
	any_created \|= replica->ip->cmds->create_probe(replica->ip, node,
	complete, force,
	data_set);
	}
	if (replica->child && (node->details == replica->node->details)) {
	any_created \|= replica->child->cmds->create_probe(replica->child,
	node, complete,
	force, data_set);
	}
	if (replica->container) {
	bool created = replica->container->cmds->create_probe(replica->container,
	node, complete,
	force, data_set);

	if(created) {
	any_created = TRUE;
	/* If we're limited to one replica per host (due to
	* the lack of an IP range probably), then we don't
	* want any of our peer containers starting until
	* we've established that no other copies are already
	* running.
	*
	* Partly this is to ensure that nreplicas_per_host is
	* observed, but also to ensure that the containers
	* don't fail to start because the necessary port
	* mappings (which won't include an IP for uniqueness)
	* are already taken
	*/

	for (GList *tIter = bundle_data->replicas;
	tIter && (bundle_data->nreplicas_per_host == 1);
	tIter = tIter->next) {
	pe__bundle_replica_t *other = tIter->data;

	if ((other != replica) && (other != NULL)
	&& (other->container != NULL)) {

	custom_action_order(replica->container,
	pcmk__op_key(replica->container->id, RSC_STATUS, 0),
	NULL, other->container,
	pcmk__op_key(other->container->id, RSC_START, 0),
	NULL,
	pe_order_optional\|pe_order_same_node,
	data_set);
	}
	}
	}
	}
	if (replica->container && replica->remote
	&& replica->remote->cmds->create_probe(replica->remote, node,
	complete, force,
	data_set)) {

	/* Do not probe the remote resource until we know where the
	* container is running. This is required for REMOTE_CONTAINER_HACK
	* to correctly probe remote resources.
	*/
	char *probe_uuid = pcmk__op_key(replica->remote->id, RSC_STATUS,
	0);
	pe_action_t *probe = find_first_action(replica->remote->actions,
	probe_uuid, NULL, node);

	free(probe_uuid);
	if (probe) {
	any_created = TRUE;
	crm_trace("Ordering %s probe on %s",
	replica->remote->id, node->details->uname);
	custom_action_order(replica->container,
	pcmk__op_key(replica->container->id, RSC_START, 0),
	NULL, replica->remote, NULL, probe,
	pe_order_probe, data_set);
	}
	}
	}
	return any_created;
	}

	void
	pcmk__bundle_append_meta(pe_resource_t rsc, xmlNode xml)
	{
	}

	void
	pcmk__bundle_log_actions(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (replica->ip) {
	LogActions(replica->ip, data_set);
	}
	if (replica->container) {
	LogActions(replica->container, data_set);
	}
	if (replica->remote) {
	LogActions(replica->remote, data_set);
	}
	if (replica->child) {
	LogActions(replica->child, data_set);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_colocation.c b/lib/pacemaker/pcmk_sched_colocation.c
	new file mode 100644
	index 0000000000..0a7d1107bc
	--- /dev/null
	+++ b/lib/pacemaker/pcmk_sched_colocation.c
	@@ -0,0 +1,800 @@
	+/*
	+ * Copyright 2004-2021 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <stdbool.h>
	+#include <glib.h>
	+
	+#include <crm/crm.h>
	+#include <crm/pengine/status.h>
	+#include <pacemaker-internal.h>
	+
	+#include "libpacemaker_private.h"
	+
	+#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	+ __rsc = pcmk__find_constraint_resource(data_set->resources, __name); \
	+ if (__rsc == NULL) { \
	+ pcmk__config_err("%s: No resource found for %s", __set, __name); \
	+ return; \
	+ } \
	+ } while(0)
	+
	+static gint
	+sort_cons_priority_lh(gconstpointer a, gconstpointer b)
	+{
	+ const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	+ const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;
	+
	+ if (a == NULL) {
	+ return 1;
	+ }
	+ if (b == NULL) {
	+ return -1;
	+ }
	+
	+ CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	+ CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
	+
	+ if (rsc_constraint1->rsc_lh->priority > rsc_constraint2->rsc_lh->priority) {
	+ return -1;
	+ }
	+
	+ if (rsc_constraint1->rsc_lh->priority < rsc_constraint2->rsc_lh->priority) {
	+ return 1;
	+ }
	+
	+ /* Process clones before primitives and groups */
	+ if (rsc_constraint1->rsc_lh->variant > rsc_constraint2->rsc_lh->variant) {
	+ return -1;
	+ } else if (rsc_constraint1->rsc_lh->variant < rsc_constraint2->rsc_lh->variant) {
	+ return 1;
	+ }
	+
	+ /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	+ * clones (probably unnecessary, but avoids having to update regression
	+ * tests)
	+ */
	+ if (rsc_constraint1->rsc_lh->variant == pe_clone) {
	+ if (pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
	+ && !pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
	+ return -1;
	+ } else if (!pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
	+ && pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
	+ return 1;
	+ }
	+ }
	+
	+ return strcmp(rsc_constraint1->rsc_lh->id, rsc_constraint2->rsc_lh->id);
	+}
	+
	+static gint
	+sort_cons_priority_rh(gconstpointer a, gconstpointer b)
	+{
	+ const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	+ const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;
	+
	+ if (a == NULL) {
	+ return 1;
	+ }
	+ if (b == NULL) {
	+ return -1;
	+ }
	+
	+ CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	+ CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
	+
	+ if (rsc_constraint1->rsc_rh->priority > rsc_constraint2->rsc_rh->priority) {
	+ return -1;
	+ }
	+
	+ if (rsc_constraint1->rsc_rh->priority < rsc_constraint2->rsc_rh->priority) {
	+ return 1;
	+ }
	+
	+ /* Process clones before primitives and groups */
	+ if (rsc_constraint1->rsc_rh->variant > rsc_constraint2->rsc_rh->variant) {
	+ return -1;
	+ } else if (rsc_constraint1->rsc_rh->variant < rsc_constraint2->rsc_rh->variant) {
	+ return 1;
	+ }
	+
	+ /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	+ * clones (probably unnecessary, but avoids having to update regression
	+ * tests)
	+ */
	+ if (rsc_constraint1->rsc_rh->variant == pe_clone) {
	+ if (pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
	+ && !pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
	+ return -1;
	+ } else if (!pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
	+ && pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
	+ return 1;
	+ }
	+ }
	+
	+ return strcmp(rsc_constraint1->rsc_rh->id, rsc_constraint2->rsc_rh->id);
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Add orderings necessary for an anti-colocation constraint
	+ */
	+static void
	+anti_colocation_order(pe_resource_t *first_rsc, int first_role,
	+ pe_resource_t *then_rsc, int then_role,
	+ pe_working_set_t *data_set)
	+{
	+ const char *first_tasks[] = { NULL, NULL };
	+ const char *then_tasks[] = { NULL, NULL };
	+
	+ /* Actions to make first_rsc lose first_role */
	+ if (first_role == RSC_ROLE_PROMOTED) {
	+ first_tasks[0] = CRMD_ACTION_DEMOTE;
	+
	+ } else {
	+ first_tasks[0] = CRMD_ACTION_STOP;
	+
	+ if (first_role == RSC_ROLE_UNPROMOTED) {
	+ first_tasks[1] = CRMD_ACTION_PROMOTE;
	+ }
	+ }
	+
	+ /* Actions to make then_rsc gain then_role */
	+ if (then_role == RSC_ROLE_PROMOTED) {
	+ then_tasks[0] = CRMD_ACTION_PROMOTE;
	+
	+ } else {
	+ then_tasks[0] = CRMD_ACTION_START;
	+
	+ if (then_role == RSC_ROLE_UNPROMOTED) {
	+ then_tasks[1] = CRMD_ACTION_DEMOTE;
	+ }
	+ }
	+
	+ for (int first_lpc = 0;
	+ (first_lpc <= 1) && (first_tasks[first_lpc] != NULL); first_lpc++) {
	+
	+ for (int then_lpc = 0;
	+ (then_lpc <= 1) && (then_tasks[then_lpc] != NULL); then_lpc++) {
	+
	+ new_rsc_order(first_rsc, first_tasks[first_lpc], then_rsc, then_tasks[then_lpc],
	+ pe_order_anti_colocation, data_set);
	+ }
	+ }
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Add a new colocation constraint to a cluster working set
	+ *
	+ * \param[in] id XML ID for this constraint
	+ * \param[in] node_attr Colocate by this attribute (or NULL for #uname)
	+ * \param[in] score Constraint score
	+ * \param[in] rsc_lh Resource to be colocated
	+ * \param[in] rsc_rh Resource to colocate \p rsc_lh with
	+ * \param[in] state_lh Current role of \p rsc_lh
	+ * \param[in] state_rh Current role of \p rsc_rh
	+ * \param[in] influence Whether colocation constraint has influence
	+ * \param[in] data_set Cluster working set to add constraint to
	+ */
	+void
	+pcmk__new_colocation(const char id, const char node_attr, int score,
	+ pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	+ const char state_lh, const char state_rh,
	+ bool influence, pe_working_set_t *data_set)
	+{
	+ pcmk__colocation_t *new_con = NULL;
	+
	+ if (score == 0) {
	+ crm_trace("Ignoring colocation '%s' because score is 0", id);
	+ return;
	+ }
	+ if ((rsc_lh == NULL) \|\| (rsc_rh == NULL)) {
	+ pcmk__config_err("Ignoring colocation '%s' because resource "
	+ "does not exist", id);
	+ return;
	+ }
	+
	+ new_con = calloc(1, sizeof(pcmk__colocation_t));
	+ if (new_con == NULL) {
	+ return;
	+ }
	+
	+ if (pcmk__str_eq(state_lh, RSC_ROLE_STARTED_S,
	+ pcmk__str_null_matches\|pcmk__str_casei)) {
	+ state_lh = RSC_ROLE_UNKNOWN_S;
	+ }
	+
	+ if (pcmk__str_eq(state_rh, RSC_ROLE_STARTED_S,
	+ pcmk__str_null_matches\|pcmk__str_casei)) {
	+ state_rh = RSC_ROLE_UNKNOWN_S;
	+ }
	+
	+ new_con->id = id;
	+ new_con->rsc_lh = rsc_lh;
	+ new_con->rsc_rh = rsc_rh;
	+ new_con->score = score;
	+ new_con->role_lh = text2role(state_lh);
	+ new_con->role_rh = text2role(state_rh);
	+ new_con->node_attribute = node_attr;
	+ new_con->influence = influence;
	+
	+ if (node_attr == NULL) {
	+ node_attr = CRM_ATTR_UNAME;
	+ }
	+
	+ pe_rsc_trace(rsc_lh, "%s ==> %s (%s %d)",
	+ rsc_lh->id, rsc_rh->id, node_attr, score);
	+
	+ rsc_lh->rsc_cons = g_list_insert_sorted(rsc_lh->rsc_cons, new_con,
	+ sort_cons_priority_rh);
	+
	+ rsc_rh->rsc_cons_lhs = g_list_insert_sorted(rsc_rh->rsc_cons_lhs, new_con,
	+ sort_cons_priority_lh);
	+
	+ data_set->colocation_constraints = g_list_append(data_set->colocation_constraints,
	+ new_con);
	+
	+ if (score <= -INFINITY) {
	+ anti_colocation_order(rsc_lh, new_con->role_lh, rsc_rh,
	+ new_con->role_rh, data_set);
	+ anti_colocation_order(rsc_rh, new_con->role_rh, rsc_lh,
	+ new_con->role_lh, data_set);
	+ }
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Return the boolean influence corresponding to configuration
	+ *
	+ * \param[in] coloc_id Colocation XML ID (for error logging)
	+ * \param[in] rsc Resource involved in constraint (for default)
	+ * \param[in] influence_s String value of influence option
	+ *
	+ * \return true if string evaluates true, false if string evaluates false,
	+ * or value of resource's critical option if string is NULL or invalid
	+ */
	+static bool
	+unpack_influence(const char coloc_id, const pe_resource_t rsc,
	+ const char *influence_s)
	+{
	+ if (influence_s != NULL) {
	+ int influence_i = 0;
	+
	+ if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	+ pcmk__config_err("Constraint '%s' has invalid value for "
	+ XML_COLOC_ATTR_INFLUENCE " (using default)",
	+ coloc_id);
	+ } else {
	+ return (influence_i != 0);
	+ }
	+ }
	+ return pcmk_is_set(rsc->flags, pe_rsc_critical);
	+}
	+
	+static void
	+unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	+ const char influence_s, pe_working_set_t data_set)
	+{
	+ xmlNode *xml_rsc = NULL;
	+ pe_resource_t *with = NULL;
	+ pe_resource_t *resource = NULL;
	+ const char *set_id = ID(set);
	+ const char *role = crm_element_value(set, "role");
	+ const char *sequential = crm_element_value(set, "sequential");
	+ const char *ordering = crm_element_value(set, "ordering");
	+ int local_score = score;
	+
	+ const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);
	+
	+ if (score_s) {
	+ local_score = char2score(score_s);
	+ }
	+ if (local_score == 0) {
	+ crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	+ coloc_id, set_id);
	+ return;
	+ }
	+
	+ if (ordering == NULL) {
	+ ordering = "group";
	+ }
	+
	+ if ((sequential != NULL) && !crm_is_true(sequential)) {
	+ return;
	+
	+ } else if ((local_score > 0)
	+ && pcmk__str_eq(ordering, "group", pcmk__str_casei)) {
	+ for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	+ if (with != NULL) {
	+ pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id);
	+ pcmk__new_colocation(set_id, NULL, local_score, resource,
	+ with, role, role,
	+ unpack_influence(coloc_id, resource,
	+ influence_s), data_set);
	+ }
	+ with = resource;
	+ }
	+
	+ } else if (local_score > 0) {
	+ pe_resource_t *last = NULL;
	+
	+ for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	+ if (last != NULL) {
	+ pe_rsc_trace(resource, "Colocating %s with %s",
	+ last->id, resource->id);
	+ pcmk__new_colocation(set_id, NULL, local_score, last,
	+ resource, role, role,
	+ unpack_influence(coloc_id, last,
	+ influence_s), data_set);
	+ }
	+
	+ last = resource;
	+ }
	+
	+ } else {
	+ /* Anti-colocating with every prior resource is
	+ * the only way to ensure the intuitive result
	+ * (i.e. that no one in the set can run with anyone else in the set)
	+ */
	+
	+ for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ xmlNode *xml_rsc_with = NULL;
	+ bool influence = true;
	+
	+ EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	+ influence = unpack_influence(coloc_id, resource, influence_s);
	+
	+ for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF);
	+ xml_rsc_with != NULL;
	+ xml_rsc_with = crm_next_same_xml(xml_rsc_with)) {
	+
	+ if (pcmk__str_eq(resource->id, ID(xml_rsc_with),
	+ pcmk__str_casei)) {
	+ break;
	+ }
	+ EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with));
	+ pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id,
	+ with->id);
	+ pcmk__new_colocation(set_id, NULL, local_score,
	+ resource, with, role, role,
	+ influence, data_set);
	+ }
	+ }
	+ }
	+}
	+
	+static void
	+colocate_rsc_sets(const char id, xmlNode set1, xmlNode *set2, int score,
	+ const char influence_s, pe_working_set_t data_set)
	+{
	+ xmlNode *xml_rsc = NULL;
	+ pe_resource_t *rsc_1 = NULL;
	+ pe_resource_t *rsc_2 = NULL;
	+
	+ const char *role_1 = crm_element_value(set1, "role");
	+ const char *role_2 = crm_element_value(set2, "role");
	+
	+ const char *sequential_1 = crm_element_value(set1, "sequential");
	+ const char *sequential_2 = crm_element_value(set2, "sequential");
	+
	+ if (score == 0) {
	+ crm_trace("Ignoring colocation '%s' between sets because score is 0",
	+ id);
	+ return;
	+ }
	+ if ((sequential_1 == NULL) \|\| crm_is_true(sequential_1)) {
	+ // Get the first one
	+ xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	+ if (xml_rsc != NULL) {
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	+ }
	+ }
	+
	+ if ((sequential_2 == NULL) \|\| crm_is_true(sequential_2)) {
	+ // Get the last one
	+ const char *rid = NULL;
	+
	+ for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ rid = ID(xml_rsc);
	+ }
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
	+ }
	+
	+ if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	+ pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	+ unpack_influence(id, rsc_1, influence_s),
	+ data_set);
	+
	+ } else if (rsc_1 != NULL) {
	+ bool influence = unpack_influence(id, rsc_1, influence_s);
	+
	+ for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	+ pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	+ role_2, influence, data_set);
	+ }
	+
	+ } else if (rsc_2 != NULL) {
	+ for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	+ pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	+ role_2,
	+ unpack_influence(id, rsc_1, influence_s),
	+ data_set);
	+ }
	+
	+ } else {
	+ for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	+ xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	+
	+ xmlNode *xml_rsc_2 = NULL;
	+ bool influence = true;
	+
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	+ influence = unpack_influence(id, rsc_1, influence_s);
	+
	+ for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	+ xml_rsc_2 != NULL;
	+ xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
	+
	+ EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	+ pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	+ role_1, role_2, influence,
	+ data_set);
	+ }
	+ }
	+ }
	+}
	+
	+static void
	+unpack_simple_colocation(xmlNode xml_obj, const char id,
	+ const char influence_s, pe_working_set_t data_set)
	+{
	+ int score_i = 0;
	+
	+ const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	+ const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	+ const char *id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	+ const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	+ const char *state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
	+ const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);
	+ const char *symmetrical = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	+
	+ // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
	+ const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
	+ const char *instance_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_INSTANCE);
	+
	+ pe_resource_t *rsc_lh = pcmk__find_constraint_resource(data_set->resources, id_lh);
	+ pe_resource_t *rsc_rh = pcmk__find_constraint_resource(data_set->resources, id_rh);
	+
	+ if (rsc_lh == NULL) {
	+ pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	+ "does not exist", id, id_lh);
	+ return;
	+
	+ } else if (rsc_rh == NULL) {
	+ pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	+ "does not exist", id, id_rh);
	+ return;
	+
	+ } else if ((instance_lh != NULL) && !pe_rsc_is_clone(rsc_lh)) {
	+ pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	+ "is not a clone but instance '%s' was requested",
	+ id, id_lh, instance_lh);
	+ return;
	+
	+ } else if ((instance_rh != NULL) && !pe_rsc_is_clone(rsc_rh)) {
	+ pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	+ "is not a clone but instance '%s' was requested",
	+ id, id_rh, instance_rh);
	+ return;
	+ }
	+
	+ if (instance_lh != NULL) {
	+ rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
	+ if (rsc_lh == NULL) {
	+ pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	+ "does not have an instance '%s'",
	+ id, id_lh, instance_lh);
	+ return;
	+ }
	+ }
	+
	+ if (instance_rh != NULL) {
	+ rsc_rh = find_clone_instance(rsc_rh, instance_rh, data_set);
	+ if (rsc_rh == NULL) {
	+ pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	+ "does not have an instance '%s'",
	+ "'%s'", id, id_rh, instance_rh);
	+ return;
	+ }
	+ }
	+
	+ if (crm_is_true(symmetrical)) {
	+ pcmk__config_warn("The colocation constraint '"
	+ XML_CONS_ATTR_SYMMETRICAL
	+ "' attribute has been removed");
	+ }
	+
	+ if (score) {
	+ score_i = char2score(score);
	+ }
	+
	+ pcmk__new_colocation(id, attr, score_i, rsc_lh, rsc_rh, state_lh, state_rh,
	+ unpack_influence(id, rsc_lh, influence_s), data_set);
	+}
	+
	+// \return Standard Pacemaker return code
	+static int
	+unpack_colocation_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	+ pe_working_set_t *data_set)
	+{
	+ const char *id = NULL;
	+ const char *id_lh = NULL;
	+ const char *id_rh = NULL;
	+ const char *state_lh = NULL;
	+ const char *state_rh = NULL;
	+
	+ pe_resource_t *rsc_lh = NULL;
	+ pe_resource_t *rsc_rh = NULL;
	+
	+ pe_tag_t *tag_lh = NULL;
	+ pe_tag_t *tag_rh = NULL;
	+
	+ xmlNode *rsc_set_lh = NULL;
	+ xmlNode *rsc_set_rh = NULL;
	+ bool any_sets = false;
	+
	+ *expanded_xml = NULL;
	+
	+ CRM_CHECK(xml_obj != NULL, return pcmk_rc_schema_validation);
	+
	+ id = ID(xml_obj);
	+ if (id == NULL) {
	+ pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	+ crm_element_name(xml_obj));
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ // Check whether there are any resource sets with template or tag references
	+ *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	+ if (*expanded_xml != NULL) {
	+ crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	+ return pcmk_rc_ok;
	+ }
	+
	+ id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	+ id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	+ if (id_lh == NULL \|\| id_rh == NULL) {
	+ return pcmk_rc_ok;
	+ }
	+
	+ if (!pcmk__valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh)) {
	+ pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	+ "valid resource or tag", id, id_lh);
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ if (!pcmk__valid_resource_or_tag(data_set, id_rh, &rsc_rh, &tag_rh)) {
	+ pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	+ "valid resource or tag", id, id_rh);
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ if (rsc_lh && rsc_rh) {
	+ /* Neither side references any template/tag. */
	+ return pcmk_rc_ok;
	+ }
	+
	+ if (tag_lh && tag_rh) {
	+ // A colocation constraint between two templates/tags makes no sense
	+ pcmk__config_err("Ignoring constraint '%s' because two templates or "
	+ "tags cannot be colocated", id);
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	+ state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
	+
	+ *expanded_xml = copy_xml(xml_obj);
	+
	+ // Convert template/tag reference in "rsc" into resource_set under constraint
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE,
	+ TRUE, data_set)) {
	+ free_xml(*expanded_xml);
	+ *expanded_xml = NULL;
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ if (rsc_set_lh != NULL) {
	+ if (state_lh != NULL) {
	+ // Move "rsc-role" into converted resource_set as "role"
	+ crm_xml_add(rsc_set_lh, "role", state_lh);
	+ xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	+ }
	+ any_sets = true;
	+ }
	+
	+ // Convert template/tag reference in "with-rsc" into resource_set under constraint
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_rh, XML_COLOC_ATTR_TARGET,
	+ TRUE, data_set)) {
	+ free_xml(*expanded_xml);
	+ *expanded_xml = NULL;
	+ return pcmk_rc_schema_validation;
	+ }
	+
	+ if (rsc_set_rh != NULL) {
	+ if (state_rh != NULL) {
	+ // Move "with-rsc-role" into converted resource_set as "role"
	+ crm_xml_add(rsc_set_rh, "role", state_rh);
	+ xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_TARGET_ROLE);
	+ }
	+ any_sets = true;
	+ }
	+
	+ if (any_sets) {
	+ crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	+ } else {
	+ free_xml(*expanded_xml);
	+ *expanded_xml = NULL;
	+ }
	+
	+ return pcmk_rc_ok;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Parse a colocation constraint from XML into a cluster working set
	+ *
	+ * \param[in] xml_obj Colocation constraint XML to unpack
	+ * \param[in] data_set Cluster working set to add constraint to
	+ */
	+void
	+pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set)
	+{
	+ int score_i = 0;
	+ xmlNode *set = NULL;
	+ xmlNode *last = NULL;
	+
	+ xmlNode *orig_xml = NULL;
	+ xmlNode *expanded_xml = NULL;
	+
	+ const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	+ const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	+ const char *influence_s = crm_element_value(xml_obj,
	+ XML_COLOC_ATTR_INFLUENCE);
	+
	+ if (score) {
	+ score_i = char2score(score);
	+ }
	+
	+ if (unpack_colocation_tags(xml_obj, &expanded_xml,
	+ data_set) != pcmk_rc_ok) {
	+ return;
	+ }
	+ if (expanded_xml) {
	+ orig_xml = xml_obj;
	+ xml_obj = expanded_xml;
	+ }
	+
	+ for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	+ set = crm_next_same_xml(set)) {
	+
	+ set = expand_idref(set, data_set->input);
	+ if (set == NULL) { // Configuration error, message already logged
	+ if (expanded_xml != NULL) {
	+ free_xml(expanded_xml);
	+ }
	+ return;
	+ }
	+
	+ unpack_colocation_set(set, score_i, id, influence_s, data_set);
	+
	+ if (last != NULL) {
	+ colocate_rsc_sets(id, last, set, score_i, influence_s, data_set);
	+ }
	+ last = set;
	+ }
	+
	+ if (expanded_xml) {
	+ free_xml(expanded_xml);
	+ xml_obj = orig_xml;
	+ }
	+
	+ if (last == NULL) {
	+ unpack_simple_colocation(xml_obj, id, influence_s, data_set);
	+ }
	+}
	+
	+static void
	+mark_start_blocked(pe_resource_t rsc, pe_resource_t reason,
	+ pe_working_set_t *data_set)
	+{
	+ char *reason_text = crm_strdup_printf("colocation with %s", reason->id);
	+
	+ for (GList *gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	+ pe_action_t action = (pe_action_t ) gIter->data;
	+
	+ if (pcmk_is_set(action->flags, pe_action_runnable)
	+ && pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	+
	+ pe__clear_action_flags(action, pe_action_runnable);
	+ pe_action_set_reason(action, reason_text, false);
	+ pcmk__block_colocated_starts(action, data_set);
	+ update_action(action, data_set);
	+ }
	+ }
	+ free(reason_text);
	+}
	+
	+/*!
	+ * \internal
	+ * \brief If a start action is unrunnable, block starts of colocated resources
	+ *
	+ * \param[in] action Action to check
	+ * \param[in] data_set Cluster working set
	+ */
	+void
	+pcmk__block_colocated_starts(pe_action_t action, pe_working_set_t data_set)
	+{
	+ GList *gIter = NULL;
	+ pe_resource_t *rsc = NULL;
	+
	+ if (!pcmk_is_set(action->flags, pe_action_runnable)
	+ && pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	+
	+ rsc = uber_parent(action->rsc);
	+ if (rsc->parent) {
	+ /* For bundles, uber_parent() returns the clone, not the bundle, so
	+ * the existence of rsc->parent implies this is a bundle.
	+ * In this case, we need the bundle resource, so that we can check
	+ * if all containers are stopped/stopping.
	+ */
	+ rsc = rsc->parent;
	+ }
	+ }
	+
	+ if ((rsc == NULL) \|\| (rsc->rsc_cons_lhs == NULL)) {
	+ return;
	+ }
	+
	+ // Block colocated starts only if all children (if any) have unrunnable starts
	+ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	+ pe_resource_t child = (pe_resource_t )gIter->data;
	+ pe_action_t *start = find_first_action(child->actions, NULL, RSC_START, NULL);
	+
	+ if ((start == NULL) \|\| pcmk_is_set(start->flags, pe_action_runnable)) {
	+ return;
	+ }
	+ }
	+
	+ for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	+ pcmk__colocation_t colocate_with = (pcmk__colocation_t ) gIter->data;
	+
	+ if (colocate_with->score == INFINITY) {
	+ mark_start_blocked(colocate_with->rsc_lh, action->rsc, data_set);
	+ }
	+ }
	+}
	diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
	index 280175bd64..14412d7c1c 100644
	--- a/lib/pacemaker/pcmk_sched_constraints.c
	+++ b/lib/pacemaker/pcmk_sched_constraints.c
	@@ -1,3009 +1,2346 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdbool.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>
	+#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	- __rsc = pe_find_constraint_resource(data_set->resources, __name); \
	+ __rsc = pcmk__find_constraint_resource(data_set->resources, __name); \
	if(__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name); \
	return FALSE; \
	} \
	} while(0)

	static pe__location_t generate_location_rule(pe_resource_t rsc,
	xmlNode *rule_xml,
	const char *discovery,
	crm_time_t *next_change,
	pe_working_set_t *data_set,
	pe_re_match_data_t *match_data);
	static void unpack_location(xmlNode xml_obj, pe_working_set_t data_set);
	-static void unpack_rsc_colocation(xmlNode xml_obj, pe_working_set_t data_set);
	static void unpack_rsc_order(xmlNode xml_obj, pe_working_set_t data_set);
	static void unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set);

	static bool
	evaluate_lifetime(xmlNode lifetime, pe_working_set_t data_set)
	{
	bool result = FALSE;
	crm_time_t *next_change = crm_time_new_undefined();

	result = pe_evaluate_rules(lifetime, NULL, data_set->now, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, data_set);
	}
	crm_time_free(next_change);
	return result;
	}

	gboolean
	unpack_constraints(xmlNode * xml_constraints, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	xmlNode *lifetime = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_constraints); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *tag = crm_element_name(xml_obj);

	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without "
	XML_ATTR_ID, tag);
	continue;
	}

	crm_trace("Unpacking %s constraint '%s'", tag, id);

	lifetime = first_named_child(xml_obj, "lifetime");
	if (lifetime) {
	pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
	"deprecated (the rules it contains should "
	"instead be direct descendents of the "
	"constraint object)", id);
	}

	if (lifetime && !evaluate_lifetime(lifetime, data_set)) {
	crm_info("Constraint %s %s is not active", tag, id);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_ORDER, tag, pcmk__str_casei)) {
	unpack_rsc_order(xml_obj, data_set);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, tag, pcmk__str_casei)) {
	- unpack_rsc_colocation(xml_obj, data_set);
	+ pcmk__unpack_colocation(xml_obj, data_set);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, tag, pcmk__str_casei)) {
	unpack_location(xml_obj, data_set);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_TICKET, tag, pcmk__str_casei)) {
	unpack_rsc_ticket(xml_obj, data_set);

	} else {
	pe_err("Unsupported constraint type: %s", tag);
	}
	}

	return TRUE;
	}

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, RSC_START, pcmk__str_casei)) {
	return RSC_STOP;

	} else if (pcmk__str_eq(action, RSC_STOP, pcmk__str_casei)) {
	return RSC_START;

	} else if (pcmk__str_eq(action, RSC_PROMOTE, pcmk__str_casei)) {
	return RSC_DEMOTE;

	} else if (pcmk__str_eq(action, RSC_DEMOTE, pcmk__str_casei)) {
	return RSC_PROMOTE;

	} else if (pcmk__str_eq(action, RSC_PROMOTED, pcmk__str_casei)) {
	return RSC_DEMOTED;

	} else if (pcmk__str_eq(action, RSC_DEMOTED, pcmk__str_casei)) {
	return RSC_PROMOTED;

	} else if (pcmk__str_eq(action, RSC_STARTED, pcmk__str_casei)) {
	return RSC_STOPPED;

	} else if (pcmk__str_eq(action, RSC_STOPPED, pcmk__str_casei)) {
	return RSC_STARTED;
	}
	crm_warn("Unknown action '%s' specified in order constraint", action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(xmlNode * xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pe_wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release (use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_casei)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_casei)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_casei)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"'%s' to Mandatory because '%s' is not valid",
	crm_str(ID(xml_obj)), kind);
	}
	return kind_e;
	}

	-static pe_resource_t *
	-pe_find_constraint_resource(GList rsc_list, const char id)
	+pe_resource_t *
	+pcmk__find_constraint_resource(GList rsc_list, const char id)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pe_resource_t *parent = rIter->data;
	pe_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
	pe_find_renamed);

	if (match != NULL) {
	if(!pcmk__str_eq(match->id, id, pcmk__str_casei)) {
	/* We found an instance of a clone instead */
	match = uber_parent(match);
	crm_debug("Found %s for %s", match->id, id);
	}
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	static gboolean
	pe_find_constraint_tag(pe_working_set_t * data_set, const char * id, pe_tag_t ** tag)
	{
	gboolean rc = FALSE;

	*tag = NULL;
	rc = g_hash_table_lookup_extended(data_set->template_rsc_sets, id,
	NULL, (gpointer*) tag);

	if (rc == FALSE) {
	rc = g_hash_table_lookup_extended(data_set->tags, id,
	NULL, (gpointer*) tag);

	if (rc == FALSE) {
	crm_warn("No template or tag named '%s'", id);
	return FALSE;

	} else if (*tag == NULL) {
	crm_warn("No resource is tagged with '%s'", id);
	return FALSE;
	}

	} else if (*tag == NULL) {
	crm_warn("No resource is derived from template '%s'", id);
	return FALSE;
	}

	return rc;
	}

	-static gboolean
	-valid_resource_or_tag(pe_working_set_t * data_set, const char * id,
	- pe_resource_t rsc, pe_tag_t tag)
	+gboolean
	+pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	+ pe_resource_t rsc, pe_tag_t tag)
	{
	gboolean rc = FALSE;

	if (rsc) {
	*rsc = NULL;
	- *rsc = pe_find_constraint_resource(data_set->resources, id);
	+ *rsc = pcmk__find_constraint_resource(data_set->resources, id);
	if (*rsc) {
	return TRUE;
	}
	}

	if (tag) {
	*tag = NULL;
	rc = pe_find_constraint_tag(data_set, id, tag);
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	const char *symmetrical_s = NULL;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	symmetrical_s = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	if (symmetrical_s == NULL) {
	symmetrical_s = parent_symmetrical_s;
	}
	if (symmetrical_s != NULL) {
	if (crm_is_true(symmetrical_s)) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static enum pe_ordering
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	enum pe_ordering flags = pe_order_none; // so we trace-log all flags set

	pe__set_order_flags(flags, pe_order_optional);

	switch (kind) {
	case pe_order_kind_optional:
	break;

	case pe_order_kind_serialize:
	pe__set_order_flags(flags, pe_order_serialize_only);
	break;

	case pe_order_kind_mandatory:
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pe_order_asymmetrical);
	break;

	case ordering_symmetric:
	pe__set_order_flags(flags, pe_order_implies_then);
	if (pcmk__strcase_any_of(first, RSC_START, RSC_PROMOTE,
	NULL)) {
	pe__set_order_flags(flags, pe_order_runnable_left);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pe_order_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number
	* \param[in] data_set Cluster working set
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pe_resource_t *
	get_ordering_resource(xmlNode xml, const char resource_attr,
	const char instance_attr, pe_working_set_t data_set)
	{
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	- rsc = pe_find_constraint_resource(data_set->resources, rsc_id);
	+ rsc = pcmk__find_constraint_resource(data_set->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id, data_set);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(pe_resource_t rsc, xmlNode xml)
	{
	if (pe_rsc_is_clone(rsc)) {
	const char *clone_min = NULL;

	clone_min = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INCARNATION_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	clone_min = crm_element_value(xml, "require-all");
	if (clone_min != NULL) {
	pe_warn_once(pe_wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!crm_is_true(clone_min)) {
	return 1;
	}
	}
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	* \param[in] data_set Cluster working set
	*/
	static void
	clone_min_ordering(const char *id,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then,
	enum pe_ordering flags, int clone_min,
	pe_working_set_t *data_set)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(CRM_OP_RELAXED_CLONE ":%s", id);
	pe_action_t *clone_min_met = get_pseudo_op(task, data_set);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pe_action_requires_any);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *rIter = rsc_first->children; rIter != NULL;
	rIter = rIter->next) {

	pe_resource_t *child = rIter->data;

	custom_action_order(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	data_set);
	}

	// Order "then" action after the pseudo-action (if runnable)
	custom_action_order(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] data_set Cluster working set
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then,
	pe_working_set_t *data_set)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	enum pe_ordering flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pe_order_implies_first, flags);
	new_rsc_order(rsc_then, action_then, rsc_first, action_first, flags,
	data_set);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
	{
	pe_resource_t *rsc_then = NULL;
	pe_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	enum pe_ordering cons_weight = pe_order_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	data_set);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	data_set);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = RSC_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	cons_weight = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pe_order_implies_then, cons_weight);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	cons_weight, min_required_before, data_set);
	} else {
	new_rsc_order(rsc_first, action_first, rsc_then, action_then,
	cons_weight, data_set);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then, data_set);
	}
	}

	/*!
	* \internal
	* \brief Replace any resource tags with equivalent resource_ref entries
	*
	* If a given constraint has resource sets, check each set for resource_ref
	* entries that list tags rather than resource IDs, and replace any found with
	* resource_ref entries for the corresponding resource IDs.
	*
	* \param[in] xml_obj Constraint XML
	* \param[in] data_set Cluster working set
	*
	* \return Equivalent XML with resource tags replaced (or NULL if none)
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	-static xmlNode *
	-expand_tags_in_sets(xmlNode xml_obj, pe_working_set_t data_set)
	+xmlNode *
	+pcmk__expand_tags_in_sets(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *new_xml = NULL;
	bool any_refs = false;

	// Short-circuit if there are no sets
	if (first_named_child(xml_obj, XML_CONS_TAG_RSC_SET) == NULL) {
	return NULL;
	}

	new_xml = copy_xml(xml_obj);

	for (xmlNode *set = first_named_child(new_xml, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	GList *tag_refs = NULL;
	GList *gIter = NULL;

	for (xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	pe_resource_t *rsc = NULL;
	pe_tag_t *tag = NULL;

	- if (!valid_resource_or_tag(data_set, ID(xml_rsc), &rsc, &tag)) {
	+ if (!pcmk__valid_resource_or_tag(data_set, ID(xml_rsc), &rsc,
	+ &tag)) {
	pcmk__config_err("Ignoring resource sets for constraint '%s' "
	"because '%s' is not a valid resource or tag",
	ID(xml_obj), ID(xml_rsc));
	free_xml(new_xml);
	return NULL;

	} else if (rsc) {
	continue;

	} else if (tag) {
	/* The resource_ref under the resource_set references a template/tag */
	xmlNode *last_ref = xml_rsc;

	/* A sample:

	Original XML:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="tag1"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	Now we are appending rsc2 and rsc3 which are tagged with tag1 right after it:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="tag1"/>
	<resource_ref id="rsc2"/>
	<resource_ref id="rsc3"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	*/

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char obj_ref = (const char ) gIter->data;
	xmlNode *new_rsc_ref = NULL;

	new_rsc_ref = xmlNewDocRawNode(getDocPtr(set), NULL,
	(pcmkXmlStr) XML_TAG_RESOURCE_REF, NULL);
	crm_xml_add(new_rsc_ref, XML_ATTR_ID, obj_ref);
	xmlAddNextSibling(last_ref, new_rsc_ref);

	last_ref = new_rsc_ref;
	}

	any_refs = true;

	/* Freeing the resource_ref now would break the XML child
	* iteration, so just remember it for freeing later.
	*/
	tag_refs = g_list_append(tag_refs, xml_rsc);
	}
	}

	/* Now free '<resource_ref id="tag1"/>', and finally get:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="rsc2"/>
	<resource_ref id="rsc3"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	*/
	for (gIter = tag_refs; gIter != NULL; gIter = gIter->next) {
	xmlNode *tag_ref = gIter->data;

	free_xml(tag_ref);
	}
	g_list_free(tag_refs);
	}

	if (!any_refs) {
	free_xml(new_xml);
	new_xml = NULL;
	}
	return new_xml;
	}

	-static gboolean
	-tag_to_set(xmlNode * xml_obj, xmlNode ** rsc_set, const char * attr,
	- gboolean convert_rsc, pe_working_set_t * data_set)
	+gboolean
	+pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	+ gboolean convert_rsc, pe_working_set_t *data_set)
	{
	const char *cons_id = NULL;
	const char *id = NULL;

	pe_resource_t *rsc = NULL;
	pe_tag_t *tag = NULL;

	*rsc_set = NULL;

	CRM_CHECK((xml_obj != NULL) && (attr != NULL), return FALSE);

	cons_id = ID(xml_obj);
	if (cons_id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return FALSE;
	}

	id = crm_element_value(xml_obj, attr);
	if (id == NULL) {
	return TRUE;
	}

	- if (valid_resource_or_tag(data_set, id, &rsc, &tag) == FALSE) {
	+ if (!pcmk__valid_resource_or_tag(data_set, id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", cons_id, id);
	return FALSE;

	} else if (tag) {
	GList *gIter = NULL;

	/* A template/tag is referenced by the "attr" attribute (first, then, rsc or with-rsc).
	Add the template/tag's corresponding "resource_set" which contains the resources derived
	from it or tagged with it under the constraint. */
	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char obj_ref = (const char ) gIter->data;
	xmlNode *rsc_ref = NULL;

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, obj_ref);
	}

	/* Set sequential="false" for the resource_set */
	crm_xml_add(*rsc_set, "sequential", XML_BOOLEAN_FALSE);

	} else if (rsc && convert_rsc) {
	/* Even a regular resource is referenced by "attr", convert it into a resource_set.
	Because the other side of the constraint could be a template/tag reference. */
	xmlNode *rsc_ref = NULL;

	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, id);

	} else {
	return TRUE;
	}

	/* Remove the "attr" attribute referencing the template/tag */
	if (*rsc_set) {
	xml_remove_prop(xml_obj, attr);
	}

	return TRUE;
	}

	static void unpack_rsc_location(xmlNode xml_obj, pe_resource_t rsc_lh,
	const char role, const char score,
	pe_working_set_t *data_set,
	pe_re_match_data_t *match_data);

	static void
	unpack_simple_location(xmlNode xml_obj, pe_working_set_t data_set)
	{
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);

	if(value) {
	- pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, value);
	+ pe_resource_t *rsc_lh = pcmk__find_constraint_resource(data_set->resources, value);

	unpack_rsc_location(xml_obj, rsc_lh, NULL, NULL, data_set, NULL);
	}

	value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
	if(value) {
	regex_t *r_patt = calloc(1, sizeof(regex_t));
	bool invert = FALSE;
	GList *rIter = NULL;

	if(value[0] == '!') {
	value++;
	invert = TRUE;
	}

	if (regcomp(r_patt, value, REG_EXTENDED)) {
	pcmk__config_err("Ignoring constraint '%s' because "
	XML_LOC_ATTR_SOURCE_PATTERN
	" has invalid value '%s'", id, value);
	regfree(r_patt);
	free(r_patt);
	return;
	}

	for (rIter = data_set->resources; rIter; rIter = rIter->next) {
	pe_resource_t *r = rIter->data;
	int nregs = 0;
	regmatch_t *pmatch = NULL;
	int status;

	if(r_patt->re_nsub > 0) {
	nregs = r_patt->re_nsub + 1;
	} else {
	nregs = 1;
	}
	pmatch = calloc(nregs, sizeof(regmatch_t));

	status = regexec(r_patt, r->id, nregs, pmatch, 0);

	if(invert == FALSE && status == 0) {
	pe_re_match_data_t re_match_data = {
	.string = r->id,
	.nregs = nregs,
	.pmatch = pmatch
	};

	crm_debug("'%s' matched '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, &re_match_data);

	} else if (invert && (status != 0)) {
	crm_debug("'%s' is an inverted match of '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, NULL);

	} else {
	crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
	}

	free(pmatch);
	}

	regfree(r_patt);
	free(r_patt);
	}
	}

	static void
	unpack_rsc_location(xmlNode xml_obj, pe_resource_t rsc_lh, const char *role,
	const char score, pe_working_set_t data_set,
	pe_re_match_data_t *re_match_data)
	{
	pe__location_t *location = NULL;
	const char *id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
	const char *discovery = crm_element_value(xml_obj, XML_LOCATION_ATTR_DISCOVERY);

	if (rsc_lh == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, id_lh);
	return;
	}

	if (score == NULL) {
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	}

	if (node != NULL && score != NULL) {
	int score_i = char2score(score);
	pe_node_t *match = pe_find_node(data_set->nodes, node);

	if (!match) {
	return;
	}
	location = rsc2node_new(id, rsc_lh, score_i, discovery, match, data_set);

	} else {
	bool empty = TRUE;
	crm_time_t *next_change = crm_time_new_undefined();

	/* This loop is logically parallel to pe_evaluate_rules(), except
	* instead of checking whether any rule is active, we set up location
	* constraints for each active rule.
	*/
	for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
	rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
	empty = FALSE;
	crm_trace("Unpacking %s/%s", id, ID(rule_xml));
	generate_location_rule(rsc_lh, rule_xml, discovery, next_change,
	data_set, re_match_data);
	}

	if (empty) {
	pcmk__config_err("Ignoring constraint '%s' because it contains "
	"no rules", id);
	}

	/* If there is a point in the future when the evaluation of a rule will
	* change, make sure the scheduler is re-run by that time.
	*/
	if (crm_time_is_defined(next_change)) {
	time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(t, data_set);
	}
	crm_time_free(next_change);
	return;
	}

	if (role == NULL) {
	role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
	}

	if (location && role) {
	if (text2role(role) == RSC_ROLE_UNKNOWN) {
	pe_err("Invalid constraint %s: Bad role %s", id, role);
	return;

	} else {
	enum rsc_role_e r = text2role(role);
	switch(r) {
	case RSC_ROLE_UNKNOWN:
	case RSC_ROLE_STARTED:
	case RSC_ROLE_UNPROMOTED:
	/* Applies to all */
	location->role_filter = RSC_ROLE_UNKNOWN;
	break;
	default:
	location->role_filter = r;
	break;
	}
	}
	}
	}

	static gboolean
	unpack_location_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *id_lh = NULL;
	const char *state_lh = NULL;

	pe_resource_t *rsc_lh = NULL;

	pe_tag_t *tag_lh = NULL;

	xmlNode *rsc_set_lh = NULL;

	CRM_CHECK(xml_obj != NULL, return FALSE);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return FALSE;
	}

	// Check whether there are any resource sets with template or tag references
	- *expanded_xml = expand_tags_in_sets(xml_obj, data_set);
	+ *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");
	return TRUE;
	}

	id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	if (id_lh == NULL) {
	return TRUE;
	}

	- if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
	+ if (!pcmk__valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, id_lh);
	return FALSE;

	} else if (rsc_lh) {
	/* No template is referenced. */
	return TRUE;
	}

	state_lh = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	/* Convert the template/tag reference in "rsc" into a resource_set under the rsc_location constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_lh, XML_LOC_ATTR_SOURCE, FALSE,
	- data_set)) {
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_lh, XML_LOC_ATTR_SOURCE,
	+ FALSE, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}

	if (rsc_set_lh) {
	if (state_lh) {
	/* A "rsc-role" is specified.
	Move it into the converted resource_set as a "role"" attribute. */
	crm_xml_add(rsc_set_lh, "role", state_lh);
	xml_remove_prop(*expanded_xml, XML_RULE_ATTR_ROLE);
	}
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");

	} else {
	/* No sets */
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return TRUE;
	}

	static gboolean
	unpack_location_set(xmlNode * location, xmlNode * set, pe_working_set_t * data_set)
	{
	xmlNode *xml_rsc = NULL;
	pe_resource_t *resource = NULL;
	const char *set_id;
	const char *role;
	const char *local_score;

	CRM_CHECK(set != NULL, return FALSE);

	set_id = ID(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
	XML_ATTR_ID " in constraint '%s'",
	crm_str(ID(location)));
	return FALSE;
	}

	role = crm_element_value(set, "role");
	local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	unpack_rsc_location(location, resource, role, local_score, data_set, NULL);
	}

	return TRUE;
	}

	static void
	unpack_location(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	gboolean any_sets = FALSE;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	if (unpack_location_tags(xml_obj, &expanded_xml, data_set) == FALSE) {
	return;
	}

	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	any_sets = TRUE;
	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| !unpack_location_set(xml_obj, set, data_set)) {
	if (expanded_xml) {
	free_xml(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (any_sets == FALSE) {
	unpack_simple_location(xml_obj, data_set);
	}
	}

	static int
	get_node_score(const char rule, const char score, gboolean raw, pe_node_t * node, pe_resource_t *rsc)
	{
	int score_f = 0;

	if (score == NULL) {
	pe_err("Rule %s: no score specified. Assuming 0.", rule);

	} else if (raw) {
	score_f = char2score(score);

	} else {
	const char *attr_score = pe_node_attribute_calculated(node, score, rsc);

	if (attr_score == NULL) {
	crm_debug("Rule %s: node %s did not have a value for %s",
	rule, node->details->uname, score);
	score_f = -INFINITY;

	} else {
	crm_debug("Rule %s: node %s had value %s for %s",
	rule, node->details->uname, attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}

	static pe__location_t *
	generate_location_rule(pe_resource_t rsc, xmlNode rule_xml,
	const char discovery, crm_time_t next_change,
	pe_working_set_t *data_set,
	pe_re_match_data_t *re_match_data)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role = NULL;

	GList *gIter = NULL;
	GList *match_L = NULL;

	gboolean do_and = TRUE;
	gboolean accept = TRUE;
	gboolean raw_score = TRUE;
	gboolean score_allocated = FALSE;

	pe__location_t *location_rule = NULL;

	rule_xml = expand_idref(rule_xml, data_set->input);
	if (rule_xml == NULL) {
	return NULL;
	}

	rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
	boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
	role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);

	crm_trace("Processing rule: %s", rule_id);

	if (role != NULL && text2role(role) == RSC_ROLE_UNKNOWN) {
	pe_err("Bad role specified for %s: %s", rule_id, role);
	return NULL;
	}

	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
	if (score == NULL) {
	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
	if (score != NULL) {
	raw_score = FALSE;
	}
	}
	if (pcmk__str_eq(boolean, "or", pcmk__str_casei)) {
	do_and = FALSE;
	}

	location_rule = rsc2node_new(rule_id, rsc, 0, discovery, NULL, data_set);

	if (location_rule == NULL) {
	return NULL;
	}

	if ((re_match_data != NULL) && (re_match_data->nregs > 0)
	&& (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {

	char *result = pe_expand_re_matches(score, re_match_data);

	if (result != NULL) {
	score = result;
	score_allocated = TRUE;
	}
	}

	if (role != NULL) {
	crm_trace("Setting role filter: %s", role);
	location_rule->role_filter = text2role(role);
	if (location_rule->role_filter == RSC_ROLE_UNPROMOTED) {
	/* Any promotable clone cannot be promoted without being in the
	* unpromoted role first. Ergo, any constraint for the unpromoted
	* role applies to every role.
	*/
	location_rule->role_filter = RSC_ROLE_UNKNOWN;
	}
	}
	if (do_and) {
	GList *gIter = NULL;

	match_L = pcmk__copy_node_list(data_set->nodes, true);
	for (gIter = match_L; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int score_f = 0;
	pe_node_t node = (pe_node_t ) gIter->data;
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = pe_rsc_params(rsc, node, data_set),
	.meta = rsc->meta,
	};

	accept = pe_test_rule(rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN,
	data_set->now, next_change, &match_data);

	crm_trace("Rule %s %s on %s", ID(rule_xml), accept ? "passed" : "failed",
	node->details->uname);

	score_f = get_node_score(rule_id, score, raw_score, node, rsc);

	if (accept) {
	pe_node_t *local = pe_find_node_id(match_L, node->details->id);

	if (local == NULL && do_and) {
	continue;

	} else if (local == NULL) {
	local = pe__copy_node(node);
	match_L = g_list_append(match_L, local);
	}

	if (do_and == FALSE) {
	local->weight = pe__add_scores(local->weight, score_f);
	}
	crm_trace("node %s now has weight %d", node->details->uname, local->weight);

	} else if (do_and && !accept) {
	/* remove it */
	pe_node_t *delete = pe_find_node_id(match_L, node->details->id);

	if (delete != NULL) {
	match_L = g_list_remove(match_L, delete);
	crm_trace("node %s did not match", node->details->uname);
	}
	free(delete);
	}
	}

	if (score_allocated == TRUE) {
	free((char *)score);
	}

	location_rule->node_list_rh = match_L;
	if (location_rule->node_list_rh == NULL) {
	crm_trace("No matching nodes for rule %s", rule_id);
	return NULL;
	}

	crm_trace("%s: %d nodes matched", rule_id, g_list_length(location_rule->node_list_rh));
	return location_rule;
	}

	-static gint
	-sort_cons_priority_lh(gconstpointer a, gconstpointer b)
	-{
	- const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	- const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;
	-
	- if (a == NULL) {
	- return 1;
	- }
	- if (b == NULL) {
	- return -1;
	- }
	-
	- CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	- CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
	-
	- if (rsc_constraint1->rsc_lh->priority > rsc_constraint2->rsc_lh->priority) {
	- return -1;
	- }
	-
	- if (rsc_constraint1->rsc_lh->priority < rsc_constraint2->rsc_lh->priority) {
	- return 1;
	- }
	-
	- /* Process clones before primitives and groups */
	- if (rsc_constraint1->rsc_lh->variant > rsc_constraint2->rsc_lh->variant) {
	- return -1;
	- } else if (rsc_constraint1->rsc_lh->variant < rsc_constraint2->rsc_lh->variant) {
	- return 1;
	- }
	-
	- /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	- * clones (probably unnecessary, but avoids having to update regression
	- * tests)
	- */
	- if (rsc_constraint1->rsc_lh->variant == pe_clone) {
	- if (pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
	- && !pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
	- return -1;
	- } else if (!pcmk_is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
	- && pcmk_is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
	- return 1;
	- }
	- }
	-
	- return strcmp(rsc_constraint1->rsc_lh->id, rsc_constraint2->rsc_lh->id);
	-}
	-
	-static gint
	-sort_cons_priority_rh(gconstpointer a, gconstpointer b)
	-{
	- const pcmk__colocation_t rsc_constraint1 = (const pcmk__colocation_t ) a;
	- const pcmk__colocation_t rsc_constraint2 = (const pcmk__colocation_t ) b;
	-
	- if (a == NULL) {
	- return 1;
	- }
	- if (b == NULL) {
	- return -1;
	- }
	-
	- CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
	- CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
	-
	- if (rsc_constraint1->rsc_rh->priority > rsc_constraint2->rsc_rh->priority) {
	- return -1;
	- }
	-
	- if (rsc_constraint1->rsc_rh->priority < rsc_constraint2->rsc_rh->priority) {
	- return 1;
	- }
	-
	- /* Process clones before primitives and groups */
	- if (rsc_constraint1->rsc_rh->variant > rsc_constraint2->rsc_rh->variant) {
	- return -1;
	- } else if (rsc_constraint1->rsc_rh->variant < rsc_constraint2->rsc_rh->variant) {
	- return 1;
	- }
	-
	- /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
	- * clones (probably unnecessary, but avoids having to update regression
	- * tests)
	- */
	- if (rsc_constraint1->rsc_rh->variant == pe_clone) {
	- if (pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
	- && !pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
	- return -1;
	- } else if (!pcmk_is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
	- && pcmk_is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
	- return 1;
	- }
	- }
	-
	- return strcmp(rsc_constraint1->rsc_rh->id, rsc_constraint2->rsc_rh->id);
	-}
	-
	-static void
	-anti_colocation_order(pe_resource_t * first_rsc, int first_role,
	- pe_resource_t * then_rsc, int then_role,
	- pe_working_set_t * data_set)
	-{
	- const char *first_tasks[] = { NULL, NULL };
	- const char *then_tasks[] = { NULL, NULL };
	- int first_lpc = 0;
	- int then_lpc = 0;
	-
	- /* Actions to make first_rsc lose first_role */
	- if (first_role == RSC_ROLE_PROMOTED) {
	- first_tasks[0] = CRMD_ACTION_DEMOTE;
	-
	- } else {
	- first_tasks[0] = CRMD_ACTION_STOP;
	-
	- if (first_role == RSC_ROLE_UNPROMOTED) {
	- first_tasks[1] = CRMD_ACTION_PROMOTE;
	- }
	- }
	-
	- /* Actions to make then_rsc gain then_role */
	- if (then_role == RSC_ROLE_PROMOTED) {
	- then_tasks[0] = CRMD_ACTION_PROMOTE;
	-
	- } else {
	- then_tasks[0] = CRMD_ACTION_START;
	-
	- if (then_role == RSC_ROLE_UNPROMOTED) {
	- then_tasks[1] = CRMD_ACTION_DEMOTE;
	- }
	- }
	-
	- for (first_lpc = 0; first_lpc <= 1 && first_tasks[first_lpc] != NULL; first_lpc++) {
	- for (then_lpc = 0; then_lpc <= 1 && then_tasks[then_lpc] != NULL; then_lpc++) {
	- new_rsc_order(first_rsc, first_tasks[first_lpc], then_rsc, then_tasks[then_lpc],
	- pe_order_anti_colocation, data_set);
	- }
	- }
	-}
	-
	-void
	-pcmk__new_colocation(const char id, const char node_attr, int score,
	- pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	- const char state_lh, const char state_rh,
	- bool influence, pe_working_set_t *data_set)
	-{
	- pcmk__colocation_t *new_con = NULL;
	-
	- if (score == 0) {
	- crm_trace("Ignoring colocation '%s' because score is 0", id);
	- return;
	- }
	- if ((rsc_lh == NULL) \|\| (rsc_rh == NULL)) {
	- pcmk__config_err("Ignoring colocation '%s' because resource "
	- "does not exist", id);
	- return;
	- }
	-
	- new_con = calloc(1, sizeof(pcmk__colocation_t));
	- if (new_con == NULL) {
	- return;
	- }
	-
	- if (pcmk__str_eq(state_lh, RSC_ROLE_STARTED_S, pcmk__str_null_matches \| pcmk__str_casei)) {
	- state_lh = RSC_ROLE_UNKNOWN_S;
	- }
	-
	- if (pcmk__str_eq(state_rh, RSC_ROLE_STARTED_S, pcmk__str_null_matches \| pcmk__str_casei)) {
	- state_rh = RSC_ROLE_UNKNOWN_S;
	- }
	-
	- new_con->id = id;
	- new_con->rsc_lh = rsc_lh;
	- new_con->rsc_rh = rsc_rh;
	- new_con->score = score;
	- new_con->role_lh = text2role(state_lh);
	- new_con->role_rh = text2role(state_rh);
	- new_con->node_attribute = node_attr;
	- new_con->influence = influence;
	-
	- if (node_attr == NULL) {
	- node_attr = CRM_ATTR_UNAME;
	- }
	-
	- pe_rsc_trace(rsc_lh, "%s ==> %s (%s %d)", rsc_lh->id, rsc_rh->id, node_attr, score);
	-
	- rsc_lh->rsc_cons = g_list_insert_sorted(rsc_lh->rsc_cons, new_con, sort_cons_priority_rh);
	-
	- rsc_rh->rsc_cons_lhs =
	- g_list_insert_sorted(rsc_rh->rsc_cons_lhs, new_con, sort_cons_priority_lh);
	-
	- data_set->colocation_constraints = g_list_append(data_set->colocation_constraints, new_con);
	-
	- if (score <= -INFINITY) {
	- anti_colocation_order(rsc_lh, new_con->role_lh, rsc_rh, new_con->role_rh, data_set);
	- anti_colocation_order(rsc_rh, new_con->role_rh, rsc_lh, new_con->role_lh, data_set);
	- }
	-}
	-
	/* LHS before RHS */
	int
	new_rsc_order(pe_resource_t * lh_rsc, const char *lh_task,
	pe_resource_t * rh_rsc, const char *rh_task,
	enum pe_ordering type, pe_working_set_t * data_set)
	{
	char *lh_key = NULL;
	char *rh_key = NULL;

	CRM_CHECK(lh_rsc != NULL, return -1);
	CRM_CHECK(lh_task != NULL, return -1);
	CRM_CHECK(rh_rsc != NULL, return -1);
	CRM_CHECK(rh_task != NULL, return -1);

	lh_key = pcmk__op_key(lh_rsc->id, lh_task, 0);
	rh_key = pcmk__op_key(rh_rsc->id, rh_task, 0);

	return custom_action_order(lh_rsc, lh_key, NULL, rh_rsc, rh_key, NULL, type, data_set);
	}

	static char *
	task_from_action_or_key(pe_action_t action, const char key)
	{
	char *res = NULL;

	if (action) {
	res = strdup(action->task);
	} else if (key) {
	parse_op_key(key, NULL, &res, NULL);
	}
	return res;
	}

	/* when order constraints are made between two resources start and stop actions
	* those constraints have to be mirrored against the corresponding
	* migration actions to ensure start/stop ordering is preserved during
	* a migration */
	static void
	handle_migration_ordering(pe__ordering_t order, pe_working_set_t data_set)
	{
	char *lh_task = NULL;
	char *rh_task = NULL;
	gboolean rh_migratable;
	gboolean lh_migratable;

	if (order->lh_rsc == NULL \|\| order->rh_rsc == NULL) {
	return;
	} else if (order->lh_rsc == order->rh_rsc) {
	return;
	/* don't mess with those constraints built between parent
	* resources and the children */
	} else if (is_parent(order->lh_rsc, order->rh_rsc)) {
	return;
	} else if (is_parent(order->rh_rsc, order->lh_rsc)) {
	return;
	}

	lh_migratable = pcmk_is_set(order->lh_rsc->flags, pe_rsc_allow_migrate);
	rh_migratable = pcmk_is_set(order->rh_rsc->flags, pe_rsc_allow_migrate);

	/* one of them has to be migratable for
	* the migrate ordering logic to be applied */
	if (lh_migratable == FALSE && rh_migratable == FALSE) {
	return;
	}

	/* at this point we have two resources which allow migrations that have an
	* order dependency set between them. If those order dependencies involve
	* start/stop actions, we need to mirror the corresponding migrate actions
	* so order will be preserved. */
	lh_task = task_from_action_or_key(order->lh_action, order->lh_action_task);
	rh_task = task_from_action_or_key(order->rh_action, order->rh_action_task);
	if (lh_task == NULL \|\| rh_task == NULL) {
	goto cleanup_order;
	}

	if (pcmk__str_eq(lh_task, RSC_START, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_START, pcmk__str_casei)) {
	int flags = pe_order_optional;

	if (lh_migratable && rh_migratable) {
	/* A start then B start
	* A migrate_from then B migrate_to */
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_MIGRATED, 0),
	NULL, order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
	NULL, flags, data_set);
	}

	if (rh_migratable) {
	if (lh_migratable) {
	pe__set_order_flags(flags, pe_order_apply_first_non_migratable);
	}

	/* A start then B start
	* A start then B migrate_to... only if A start is not a part of a migration*/
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_START, 0),
	NULL, order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
	NULL, flags, data_set);
	}

	} else if (rh_migratable == TRUE && pcmk__str_eq(lh_task, RSC_STOP, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_STOP, pcmk__str_casei)) {
	int flags = pe_order_optional;

	if (lh_migratable) {
	pe__set_order_flags(flags, pe_order_apply_first_non_migratable);
	}

	/* rh side is at the bottom of the stack during a stop. If we have a constraint
	* stop B then stop A, if B is migrating via stop/start, and A is migrating using migration actions,
	* we need to enforce that A's migrate_to action occurs after B's stop action. */
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0), NULL,
	order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
	NULL, flags, data_set);

	/* We need to build the stop constraint against migrate_from as well
	* to account for partial migrations. */
	if (order->rh_rsc->partial_migration_target) {
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0),
	NULL, order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
	NULL, flags, data_set);
	}

	} else if (pcmk__str_eq(lh_task, RSC_PROMOTE, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_START, pcmk__str_casei)) {
	int flags = pe_order_optional;

	if (rh_migratable) {
	/* A promote then B start
	* A promote then B migrate_to */
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_PROMOTE, 0),
	NULL, order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
	NULL, flags, data_set);
	}

	} else if (pcmk__str_eq(lh_task, RSC_DEMOTE, pcmk__str_casei) && pcmk__str_eq(rh_task, RSC_STOP, pcmk__str_casei)) {
	int flags = pe_order_optional;

	if (rh_migratable) {
	/* A demote then B stop
	* A demote then B migrate_to */
	custom_action_order(order->lh_rsc, pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0), NULL,
	order->rh_rsc, pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0), NULL,
	flags, data_set);

	/* We need to build the demote constraint against migrate_from as well
	* to account for partial migrations. */
	if (order->rh_rsc->partial_migration_target) {
	custom_action_order(order->lh_rsc,
	pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0),
	NULL, order->rh_rsc,
	pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
	NULL, flags, data_set);
	}
	}
	}

	cleanup_order:
	free(lh_task);
	free(rh_task);
	}

	/* LHS before RHS */
	int
	custom_action_order(pe_resource_t * lh_rsc, char lh_action_task, pe_action_t lh_action,
	pe_resource_t * rh_rsc, char rh_action_task, pe_action_t rh_action,
	enum pe_ordering type, pe_working_set_t * data_set)
	{
	pe__ordering_t *order = NULL;

	if (lh_rsc == NULL && lh_action) {
	lh_rsc = lh_action->rsc;
	}
	if (rh_rsc == NULL && rh_action) {
	rh_rsc = rh_action->rsc;
	}

	if ((lh_action == NULL && lh_rsc == NULL)
	\|\| (rh_action == NULL && rh_rsc == NULL)) {
	crm_err("Invalid ordering (bug?)");
	free(lh_action_task);
	free(rh_action_task);
	return -1;
	}

	order = calloc(1, sizeof(pe__ordering_t));

	order->id = data_set->order_id++;
	order->type = type;
	order->lh_rsc = lh_rsc;
	order->rh_rsc = rh_rsc;
	order->lh_action = lh_action;
	order->rh_action = rh_action;
	order->lh_action_task = lh_action_task;
	order->rh_action_task = rh_action_task;

	if (order->lh_action_task == NULL && lh_action) {
	order->lh_action_task = strdup(lh_action->uuid);
	}

	if (order->rh_action_task == NULL && rh_action) {
	order->rh_action_task = strdup(rh_action->uuid);
	}

	if (order->lh_rsc == NULL && lh_action) {
	order->lh_rsc = lh_action->rsc;
	}

	if (order->rh_rsc == NULL && rh_action) {
	order->rh_rsc = rh_action->rsc;
	}

	pe_rsc_trace(lh_rsc, "Created ordering %d for %s then %s",
	(data_set->order_id - 1),
	((lh_action_task == NULL)? "?" : lh_action_task),
	((rh_action_task == NULL)? "?" : rh_action_task));

	data_set->ordering_constraints = g_list_prepend(data_set->ordering_constraints, order);
	handle_migration_ordering(order, data_set);

	return order->id;
	}

	static gboolean
	unpack_order_set(xmlNode * set, enum pe_order_kind parent_kind, pe_resource_t ** rsc,
	const char *parent_symmetrical_s,
	pe_working_set_t * data_set)
	{
	xmlNode *xml_rsc = NULL;
	GList *set_iter = NULL;
	GList *resources = NULL;

	pe_resource_t *last = NULL;
	pe_resource_t *resource = NULL;

	int local_kind = parent_kind;
	gboolean sequential = FALSE;
	enum pe_ordering flags = pe_order_optional;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = RSC_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	*rsc = resource;
	goto done;
	}

	*rsc = NULL;

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	GList *gIter = NULL;

	for (gIter = set_iter; gIter != NULL; gIter = gIter->next) {
	pe_resource_t then_rsc = (pe_resource_t ) gIter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	custom_action_order(resource, strdup(key), NULL, then_rsc, then_key, NULL,
	flags, data_set);
	}

	} else if (sequential) {
	if (last != NULL) {
	new_rsc_order(last, action, resource, action, flags, data_set);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	new_rsc_order(resource, action, last, action, flags, data_set);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return TRUE;
	}

	static gboolean
	order_rsc_sets(const char id, xmlNode set1, xmlNode * set2, enum pe_order_kind kind,
	pe_working_set_t *data_set, enum ordering_symmetry symmetry)
	{

	xmlNode *xml_rsc = NULL;
	xmlNode *xml_rsc_2 = NULL;

	pe_resource_t *rsc_1 = NULL;
	pe_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	const char *sequential_1 = crm_element_value(set1, "sequential");
	const char *sequential_2 = crm_element_value(set2, "sequential");

	const char *require_all_s = crm_element_value(set1, "require-all");
	gboolean require_all = require_all_s ? crm_is_true(require_all_s) : TRUE;

	enum pe_ordering flags = pe_order_none;

	if (action_1 == NULL) {
	action_1 = RSC_START;
	};

	if (action_2 == NULL) {
	action_2 = RSC_START;
	};

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if(pcmk__str_eq(RSC_STOP, action_1, pcmk__str_casei) \|\| pcmk__str_eq(RSC_DEMOTE, action_1, pcmk__str_casei)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase
	* During shutdown neither B nor can shutdown until D is down, so simply turn require_all back on.
	*/
	require_all = TRUE;
	}

	// @TODO is action_2 correct here?
	flags = ordering_flags_for_kind(kind, action_2, symmetry);

	/* If we have an un-ordered set1, whether it is sequential or not is irrelevant in regards to set2. */
	if (!require_all) {
	char *task = crm_strdup_printf(CRM_OP_RELAXED_SET ":%s", ID(set1));
	pe_action_t *unordered_action = get_pseudo_op(task, data_set);

	free(task);
	pe__set_action_flags(unordered_action, pe_action_requires_any);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the pseudo action.
	* If any action in set1 is runnable the pseudo action will be runnable. */
	custom_action_order(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	data_set);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo action and every element in set2.
	* If the pseudo action is runnable, every action in set2 will be runnable */
	custom_action_order(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	return TRUE;
	}

	if (crm_is_true(sequential_1)) {
	if (symmetry == ordering_symmetric_inverse) {
	/* get the first one */
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	/* get the last one */
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (crm_is_true(sequential_2)) {
	if (symmetry == ordering_symmetric_inverse) {
	/* get the last one */
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	/* get the first one */
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if (rsc_1 != NULL && rsc_2 != NULL) {
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	}

	} else if (rsc_2 != NULL) {
	xmlNode *xml_rsc = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	xmlNode *xml_rsc_2 = NULL;

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
	}
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_schema_validation on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pe_working_set_t *data_set)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pe_resource_t *rsc_first = NULL;
	pe_resource_t *rsc_then = NULL;
	pe_tag_t *tag_first = NULL;
	pe_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	gboolean any_sets = FALSE;

	// Check whether there are any resource sets with template or tag references
	- *expanded_xml = expand_tags_in_sets(xml_obj, data_set);
	+ *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if (id_first == NULL \|\| id_then == NULL) {
	return pcmk_rc_ok;
	}

	- if (valid_resource_or_tag(data_set, id_first, &rsc_first, &tag_first) == FALSE) {
	+ if (!pcmk__valid_resource_or_tag(data_set, id_first, &rsc_first, &tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_schema_validation;
	}

	- if (valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then) == FALSE) {
	+ if (!pcmk__valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_schema_validation;
	}

	if (rsc_first && rsc_then) {
	/* Neither side references any template/tag. */
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	/* Convert the template/tag reference in "first" into a resource_set under the order constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST, TRUE,
	- data_set)) {
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	+ TRUE, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_schema_validation;
	}

	if (rsc_set_first) {
	if (action_first) {
	/* A "first-action" is specified.
	Move it into the converted resource_set as an "action" attribute. */
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = TRUE;
	}

	/* Convert the template/tag reference in "then" into a resource_set under the order constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN, TRUE,
	- data_set)) {
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	+ TRUE, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_schema_validation;
	}

	if (rsc_set_then) {
	if (action_then) {
	/* A "then-action" is specified.
	Move it into the converted resource_set as an "action" attribute. */
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = TRUE;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in] xml_obj Ordering constraint XML to unpack
	* \param[in,out] data_set Cluster working set
	*/
	static void
	unpack_rsc_order(xmlNode xml_obj, pe_working_set_t data_set)
	{
	pe_resource_t *rsc = NULL;

	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind, NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| !unpack_order_set(set, kind, &rsc, invert, data_set)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((last != NULL)
	&& (!order_rsc_sets(id, last, set, kind, data_set, symmetry)
	\|\| ((symmetry == ordering_symmetric)
	&& !order_rsc_sets(id, set, last, kind, data_set,
	ordering_symmetric_inverse)))) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	unpack_simple_rsc_order(xml_obj, data_set);
	}
	}

	-/*!
	- * \internal
	- * \brief Return the boolean influence corresponding to configuration
	- *
	- * \param[in] coloc_id Colocation XML ID (for error logging)
	- * \param[in] rsc Resource involved in constraint (for default)
	- * \param[in] influence_s String value of influence option
	- *
	- * \return true if string evaluates true, false if string evaluates false,
	- * or value of resource's critical option if string is NULL or invalid
	- */
	-static bool
	-unpack_influence(const char coloc_id, const pe_resource_t rsc,
	- const char *influence_s)
	-{
	- if (influence_s != NULL) {
	- int influence_i = 0;
	-
	- if (crm_str_to_boolean(influence_s, &influence_i) < 0) {
	- pcmk__config_err("Constraint '%s' has invalid value for "
	- XML_COLOC_ATTR_INFLUENCE " (using default)",
	- coloc_id);
	- } else {
	- return (influence_i == TRUE);
	- }
	- }
	- return pcmk_is_set(rsc->flags, pe_rsc_critical);
	-}
	-
	-static gboolean
	-unpack_colocation_set(xmlNode set, int score, const char coloc_id,
	- const char influence_s, pe_working_set_t data_set)
	-{
	- xmlNode *xml_rsc = NULL;
	- pe_resource_t *with = NULL;
	- pe_resource_t *resource = NULL;
	- const char *set_id = ID(set);
	- const char *role = crm_element_value(set, "role");
	- const char *sequential = crm_element_value(set, "sequential");
	- const char *ordering = crm_element_value(set, "ordering");
	- int local_score = score;
	-
	- const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);
	-
	- if (score_s) {
	- local_score = char2score(score_s);
	- }
	- if (local_score == 0) {
	- crm_trace("Ignoring colocation '%s' for set '%s' because score is 0",
	- coloc_id, set_id);
	- return TRUE;
	- }
	-
	- if(ordering == NULL) {
	- ordering = "group";
	- }
	-
	- if (sequential != NULL && crm_is_true(sequential) == FALSE) {
	- return TRUE;
	-
	- } else if ((local_score > 0)
	- && pcmk__str_eq(ordering, "group", pcmk__str_casei)) {
	- for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	- if (with != NULL) {
	- pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id);
	- pcmk__new_colocation(set_id, NULL, local_score, resource,
	- with, role, role,
	- unpack_influence(coloc_id, resource,
	- influence_s),
	- data_set);
	- }
	- with = resource;
	- }
	- } else if (local_score > 0) {
	- pe_resource_t *last = NULL;
	- for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	- if (last != NULL) {
	- pe_rsc_trace(resource, "Colocating %s with %s", last->id, resource->id);
	- pcmk__new_colocation(set_id, NULL, local_score, last,
	- resource, role, role,
	- unpack_influence(coloc_id, last,
	- influence_s),
	- data_set);
	- }
	-
	- last = resource;
	- }
	-
	- } else {
	- /* Anti-colocating with every prior resource is
	- * the only way to ensure the intuitive result
	- * (i.e. that no one in the set can run with anyone else in the set)
	- */
	-
	- for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- xmlNode *xml_rsc_with = NULL;
	- bool influence = true;
	-
	- EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	- influence = unpack_influence(coloc_id, resource, influence_s);
	-
	- for (xml_rsc_with = first_named_child(set, XML_TAG_RESOURCE_REF);
	- xml_rsc_with != NULL;
	- xml_rsc_with = crm_next_same_xml(xml_rsc_with)) {
	-
	- if (pcmk__str_eq(resource->id, ID(xml_rsc_with), pcmk__str_casei)) {
	- break;
	- }
	- EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with));
	- pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id,
	- with->id);
	- pcmk__new_colocation(set_id, NULL, local_score,
	- resource, with, role, role,
	- influence, data_set);
	- }
	- }
	- }
	-
	- return TRUE;
	-}
	-
	-static gboolean
	-colocate_rsc_sets(const char id, xmlNode set1, xmlNode * set2, int score,
	- const char influence_s, pe_working_set_t data_set)
	-{
	- xmlNode *xml_rsc = NULL;
	- pe_resource_t *rsc_1 = NULL;
	- pe_resource_t *rsc_2 = NULL;
	-
	- const char *role_1 = crm_element_value(set1, "role");
	- const char *role_2 = crm_element_value(set2, "role");
	-
	- const char *sequential_1 = crm_element_value(set1, "sequential");
	- const char *sequential_2 = crm_element_value(set2, "sequential");
	-
	- if (score == 0) {
	- crm_trace("Ignoring colocation '%s' between sets because score is 0",
	- id);
	- return TRUE;
	- }
	- if (sequential_1 == NULL \|\| crm_is_true(sequential_1)) {
	- /* get the first one */
	- xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	- if (xml_rsc != NULL) {
	- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	- }
	- }
	-
	- if (sequential_2 == NULL \|\| crm_is_true(sequential_2)) {
	- /* get the last one */
	- const char *rid = NULL;
	-
	- for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	- rid = ID(xml_rsc);
	- }
	- EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
	- }
	-
	- if (rsc_1 != NULL && rsc_2 != NULL) {
	- pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1, role_2,
	- unpack_influence(id, rsc_1, influence_s),
	- data_set);
	-
	- } else if (rsc_1 != NULL) {
	- bool influence = unpack_influence(id, rsc_1, influence_s);
	-
	- for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	- pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	- role_2, influence, data_set);
	- }
	-
	- } else if (rsc_2 != NULL) {
	- for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	- pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2, role_1,
	- role_2,
	- unpack_influence(id, rsc_1, influence_s),
	- data_set);
	- }
	-
	- } else {
	- for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	- xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
	-
	- xmlNode *xml_rsc_2 = NULL;
	- bool influence = true;
	-
	- EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	- influence = unpack_influence(id, rsc_1, influence_s);
	-
	- for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	- xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {
	-
	- EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	- pcmk__new_colocation(id, NULL, score, rsc_1, rsc_2,
	- role_1, role_2, influence,
	- data_set);
	- }
	- }
	- }
	-
	- return TRUE;
	-}
	-
	-static void
	-unpack_simple_colocation(xmlNode xml_obj, const char id,
	- const char influence_s, pe_working_set_t data_set)
	-{
	- int score_i = 0;
	-
	- const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	- const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	- const char *id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	- const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	- const char *state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
	- const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);
	- const char *symmetrical = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	-
	- // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
	- const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
	- const char *instance_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_INSTANCE);
	-
	- pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
	- pe_resource_t *rsc_rh = pe_find_constraint_resource(data_set->resources, id_rh);
	-
	- if (rsc_lh == NULL) {
	- pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	- "does not exist", id, id_lh);
	- return;
	-
	- } else if (rsc_rh == NULL) {
	- pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	- "does not exist", id, id_rh);
	- return;
	-
	- } else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
	- pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	- "is not a clone but instance '%s' was requested",
	- id, id_lh, instance_lh);
	- return;
	-
	- } else if (instance_rh && pe_rsc_is_clone(rsc_rh) == FALSE) {
	- pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	- "is not a clone but instance '%s' was requested",
	- id, id_rh, instance_rh);
	- return;
	- }
	-
	- if (instance_lh) {
	- rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
	- if (rsc_lh == NULL) {
	- pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	- "does not have an instance '%s'",
	- id, id_lh, instance_lh);
	- return;
	- }
	- }
	-
	- if (instance_rh) {
	- rsc_rh = find_clone_instance(rsc_rh, instance_rh, data_set);
	- if (rsc_rh == NULL) {
	- pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	- "does not have an instance '%s'",
	- "'%s'", id, id_rh, instance_rh);
	- return;
	- }
	- }
	-
	- if (crm_is_true(symmetrical)) {
	- pcmk__config_warn("The colocation constraint '"
	- XML_CONS_ATTR_SYMMETRICAL
	- "' attribute has been removed");
	- }
	-
	- if (score) {
	- score_i = char2score(score);
	- }
	-
	- pcmk__new_colocation(id, attr, score_i, rsc_lh, rsc_rh, state_lh, state_rh,
	- unpack_influence(id, rsc_lh, influence_s), data_set);
	-}
	-
	-static gboolean
	-unpack_colocation_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	-{
	- const char *id = NULL;
	- const char *id_lh = NULL;
	- const char *id_rh = NULL;
	- const char *state_lh = NULL;
	- const char *state_rh = NULL;
	-
	- pe_resource_t *rsc_lh = NULL;
	- pe_resource_t *rsc_rh = NULL;
	-
	- pe_tag_t *tag_lh = NULL;
	- pe_tag_t *tag_rh = NULL;
	-
	- xmlNode *rsc_set_lh = NULL;
	- xmlNode *rsc_set_rh = NULL;
	- gboolean any_sets = FALSE;
	-
	- *expanded_xml = NULL;
	-
	- CRM_CHECK(xml_obj != NULL, return FALSE);
	-
	- id = ID(xml_obj);
	- if (id == NULL) {
	- pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	- crm_element_name(xml_obj));
	- return FALSE;
	- }
	-
	- // Check whether there are any resource sets with template or tag references
	- *expanded_xml = expand_tags_in_sets(xml_obj, data_set);
	- if (*expanded_xml != NULL) {
	- crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	- return TRUE;
	- }
	-
	- id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	- id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
	- if (id_lh == NULL \|\| id_rh == NULL) {
	- return TRUE;
	- }
	-
	- if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
	- pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	- "valid resource or tag", id, id_lh);
	- return FALSE;
	- }
	-
	- if (valid_resource_or_tag(data_set, id_rh, &rsc_rh, &tag_rh) == FALSE) {
	- pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	- "valid resource or tag", id, id_rh);
	- return FALSE;
	- }
	-
	- if (rsc_lh && rsc_rh) {
	- /* Neither side references any template/tag. */
	- return TRUE;
	- }
	-
	- if (tag_lh && tag_rh) {
	- /* A colocation constraint between two templates/tags makes no sense. */
	- pcmk__config_err("Ignoring constraint '%s' because two templates or "
	- "tags cannot be colocated", id);
	- return FALSE;
	- }
	-
	- state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
	- state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
	-
	- *expanded_xml = copy_xml(xml_obj);
	-
	- /* Convert the template/tag reference in "rsc" into a resource_set under the colocation constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, TRUE,
	- data_set)) {
	- free_xml(*expanded_xml);
	- *expanded_xml = NULL;
	- return FALSE;
	- }
	-
	- if (rsc_set_lh) {
	- if (state_lh) {
	- /* A "rsc-role" is specified.
	- Move it into the converted resource_set as a "role"" attribute. */
	- crm_xml_add(rsc_set_lh, "role", state_lh);
	- xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	- }
	- any_sets = TRUE;
	- }
	-
	- /* Convert the template/tag reference in "with-rsc" into a resource_set under the colocation constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_rh, XML_COLOC_ATTR_TARGET, TRUE,
	- data_set)) {
	- free_xml(*expanded_xml);
	- *expanded_xml = NULL;
	- return FALSE;
	- }
	-
	- if (rsc_set_rh) {
	- if (state_rh) {
	- /* A "with-rsc-role" is specified.
	- Move it into the converted resource_set as a "role"" attribute. */
	- crm_xml_add(rsc_set_rh, "role", state_rh);
	- xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_TARGET_ROLE);
	- }
	- any_sets = TRUE;
	- }
	-
	- if (any_sets) {
	- crm_log_xml_trace(*expanded_xml, "Expanded rsc_colocation");
	- } else {
	- free_xml(*expanded_xml);
	- *expanded_xml = NULL;
	- }
	-
	- return TRUE;
	-}
	-
	-static void
	-unpack_rsc_colocation(xmlNode xml_obj, pe_working_set_t data_set)
	-{
	- int score_i = 0;
	- xmlNode *set = NULL;
	- xmlNode *last = NULL;
	-
	- xmlNode *orig_xml = NULL;
	- xmlNode *expanded_xml = NULL;
	-
	- const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	- const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	- const char *influence_s = crm_element_value(xml_obj,
	- XML_COLOC_ATTR_INFLUENCE);
	-
	- if (score) {
	- score_i = char2score(score);
	- }
	-
	- if (!unpack_colocation_tags(xml_obj, &expanded_xml, data_set)) {
	- return;
	- }
	- if (expanded_xml) {
	- orig_xml = xml_obj;
	- xml_obj = expanded_xml;
	- }
	-
	- for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	- set = crm_next_same_xml(set)) {
	-
	- set = expand_idref(set, data_set->input);
	- if ((set == NULL) // Configuration error, message already logged
	- \|\| !unpack_colocation_set(set, score_i, id, influence_s, data_set)
	- \|\| ((last != NULL) && !colocate_rsc_sets(id, last, set, score_i,
	- influence_s, data_set))) {
	- if (expanded_xml != NULL) {
	- free_xml(expanded_xml);
	- }
	- return;
	- }
	- last = set;
	- }
	-
	- if (expanded_xml) {
	- free_xml(expanded_xml);
	- xml_obj = orig_xml;
	- }
	-
	- if (last == NULL) {
	- unpack_simple_colocation(xml_obj, id, influence_s, data_set);
	- }
	-}
	-
	static void
	rsc_ticket_new(const char id, pe_resource_t rsc_lh, pe_ticket_t *ticket,
	const char state_lh, const char loss_policy,
	pe_working_set_t *data_set)
	{
	rsc_ticket_t *new_rsc_ticket = NULL;

	if (rsc_lh == NULL) {
	pcmk__config_err("Ignoring ticket '%s' because resource "
	"does not exist", id);
	return;
	}

	new_rsc_ticket = calloc(1, sizeof(rsc_ticket_t));
	if (new_rsc_ticket == NULL) {
	return;
	}

	if (pcmk__str_eq(state_lh, RSC_ROLE_STARTED_S, pcmk__str_null_matches \| pcmk__str_casei)) {
	state_lh = RSC_ROLE_UNKNOWN_S;
	}

	new_rsc_ticket->id = id;
	new_rsc_ticket->ticket = ticket;
	new_rsc_ticket->rsc_lh = rsc_lh;
	new_rsc_ticket->role_lh = text2role(state_lh);

	if (pcmk__str_eq(loss_policy, "fence", pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	new_rsc_ticket->loss_policy = loss_ticket_fence;
	} else {
	pcmk__config_err("Resetting '" XML_TICKET_ATTR_LOSS_POLICY
	"' for ticket '%s' to 'stop' "
	"because fencing is not configured", ticket->id);
	loss_policy = "stop";
	}
	}

	if (new_rsc_ticket->loss_policy == loss_ticket_fence) {
	crm_debug("On loss of ticket '%s': Fence the nodes running %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));

	} else if (pcmk__str_eq(loss_policy, "freeze", pcmk__str_casei)) {
	crm_debug("On loss of ticket '%s': Freeze %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));
	new_rsc_ticket->loss_policy = loss_ticket_freeze;

	} else if (pcmk__str_eq(loss_policy, "demote", pcmk__str_casei)) {
	crm_debug("On loss of ticket '%s': Demote %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));
	new_rsc_ticket->loss_policy = loss_ticket_demote;

	} else if (pcmk__str_eq(loss_policy, "stop", pcmk__str_casei)) {
	crm_debug("On loss of ticket '%s': Stop %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));
	new_rsc_ticket->loss_policy = loss_ticket_stop;

	} else {
	if (new_rsc_ticket->role_lh == RSC_ROLE_PROMOTED) {
	crm_debug("On loss of ticket '%s': Default to demote %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));
	new_rsc_ticket->loss_policy = loss_ticket_demote;

	} else {
	crm_debug("On loss of ticket '%s': Default to stop %s (%s)",
	new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
	role2text(new_rsc_ticket->role_lh));
	new_rsc_ticket->loss_policy = loss_ticket_stop;
	}
	}

	pe_rsc_trace(rsc_lh, "%s (%s) ==> %s", rsc_lh->id, role2text(new_rsc_ticket->role_lh),
	ticket->id);

	rsc_lh->rsc_tickets = g_list_append(rsc_lh->rsc_tickets, new_rsc_ticket);

	data_set->ticket_constraints = g_list_append(data_set->ticket_constraints, new_rsc_ticket);

	if (new_rsc_ticket->ticket->granted == FALSE \|\| new_rsc_ticket->ticket->standby) {
	rsc_ticket_constraint(rsc_lh, new_rsc_ticket, data_set);
	}
	}

	static gboolean
	unpack_rsc_ticket_set(xmlNode * set, pe_ticket_t * ticket, const char *loss_policy,
	pe_working_set_t * data_set)
	{
	xmlNode *xml_rsc = NULL;
	pe_resource_t *resource = NULL;
	const char *set_id = NULL;
	const char *role = NULL;

	CRM_CHECK(set != NULL, return FALSE);
	CRM_CHECK(ticket != NULL, return FALSE);

	set_id = ID(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring <" XML_CONS_TAG_RSC_SET "> without "
	XML_ATTR_ID);
	return FALSE;
	}

	role = crm_element_value(set, "role");

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
	pe_rsc_trace(resource, "Resource '%s' depends on ticket '%s'",
	resource->id, ticket->id);
	rsc_ticket_new(set_id, resource, ticket, role, loss_policy, data_set);
	}

	return TRUE;
	}

	static void
	unpack_simple_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
	const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);

	pe_ticket_t *ticket = NULL;

	const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);

	// experimental syntax from pacemaker-next (unlikely to be adopted as-is)
	const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);

	pe_resource_t *rsc_lh = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return;
	}

	if (ticket_str == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without ticket specified",
	id);
	return;
	} else {
	ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
	}

	if (ticket == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because ticket '%s' "
	"does not exist", id, ticket_str);
	return;
	}

	if (id_lh == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without resource", id);
	return;
	} else {
	- rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
	+ rsc_lh = pcmk__find_constraint_resource(data_set->resources, id_lh);
	}

	if (rsc_lh == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, id_lh);
	return;

	} else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	id, id_lh, instance_lh);
	return;
	}

	if (instance_lh) {
	rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
	if (rsc_lh == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", id, id_lh, instance_lh);
	return;
	}
	}

	rsc_ticket_new(id, rsc_lh, ticket, state_lh, loss_policy, data_set);
	}

	static gboolean
	unpack_rsc_ticket_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *id_lh = NULL;
	const char *state_lh = NULL;

	pe_resource_t *rsc_lh = NULL;
	pe_tag_t *tag_lh = NULL;

	xmlNode *rsc_set_lh = NULL;

	CRM_CHECK(xml_obj != NULL, return FALSE);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return FALSE;
	}

	// Check whether there are any resource sets with template or tag references
	- *expanded_xml = expand_tags_in_sets(xml_obj, data_set);
	+ *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_ticket");
	return TRUE;
	}

	id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
	if (id_lh == NULL) {
	return TRUE;
	}

	- if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
	+ if (!pcmk__valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, id_lh);
	return FALSE;

	} else if (rsc_lh) {
	/* No template/tag is referenced. */
	return TRUE;
	}

	state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	/* Convert the template/tag reference in "rsc" into a resource_set under the rsc_ticket constraint. */
	- if (!tag_to_set(*expanded_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, FALSE,
	- data_set)) {
	+ if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE,
	+ FALSE, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}

	if (rsc_set_lh) {
	if (state_lh) {
	/* A "rsc-role" is specified.
	Move it into the converted resource_set as a "role"" attribute. */
	crm_xml_add(rsc_set_lh, "role", state_lh);
	xml_remove_prop(*expanded_xml, XML_COLOC_ATTR_SOURCE_ROLE);
	}
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return TRUE;
	}

	static void
	unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	gboolean any_sets = FALSE;

	const char *id = NULL;
	const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
	const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);

	pe_ticket_t *ticket = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	gboolean rc = TRUE;

	CRM_CHECK(xml_obj != NULL, return);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return;
	}

	if (data_set->tickets == NULL) {
	data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	if (ticket_str == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without ticket", id);
	return;
	} else {
	ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
	}

	if (ticket == NULL) {
	ticket = ticket_new(ticket_str, data_set);
	if (ticket == NULL) {
	return;
	}
	}

	rc = unpack_rsc_ticket_tags(xml_obj, &expanded_xml, data_set);
	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;

	} else if (rc == FALSE) {
	return;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	any_sets = TRUE;
	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| !unpack_rsc_ticket_set(set, ticket, loss_policy, data_set)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (any_sets == FALSE) {
	unpack_simple_rsc_ticket(xml_obj, data_set);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c
	index c22a560e1b..cfa3248a62 100644
	--- a/lib/pacemaker/pcmk_sched_group.c
	+++ b/lib/pacemaker/pcmk_sched_group.c
	@@ -1,576 +1,579 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>

	#include <pacemaker-internal.h>

	+#include "libpacemaker_private.h"
	+
	#define VARIANT_GROUP 1
	#include <lib/pengine/variant.h>

	/*!
	* \internal
	* \brief Expand a group's colocations to its members
	*
	* \param[in,out] rsc Group resource
	*/
	static void
	expand_group_colocations(pe_resource_t *rsc)
	{
	group_variant_data_t *group_data = NULL;
	pe_resource_t *member = NULL;
	bool any_unmanaged = false;

	get_group_variant_data(group_data, rsc);

	// Treat "group with R" colocations as "first member with R"
	member = group_data->first_child;
	member->rsc_cons = g_list_concat(member->rsc_cons, rsc->rsc_cons);


	/* The above works for the whole group because each group member is
	* colocated with the previous one.
	*
	* However, there is a special case when a group has a mandatory colocation
	- * with a resource that can't start. In that case, update_colo_start_chain()
	- * will ensure that dependent resources in mandatory colocations (i.e. the
	- * first member for groups) can't start either. But if any group member is
	- * unmanaged and already started, the internal group colocations are no
	- * longer sufficient to make that apply to later members.
	+ * with a resource that can't start. In that case,
	+ * pcmk__block_colocated_starts() will ensure that dependent resources in
	+ * mandatory colocations (i.e. the first member for groups) can't start
	+ * either. But if any group member is unmanaged and already started, the
	+ * internal group colocations are no longer sufficient to make that apply to
	+ * later members.
	*
	* To handle that case, add mandatory colocations to each member after the
	* first.
	*/
	any_unmanaged = !pcmk_is_set(member->flags, pe_rsc_managed);
	for (GList *item = rsc->children->next; item != NULL; item = item->next) {
	member = item->data;
	if (any_unmanaged) {
	for (GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
	cons_iter = cons_iter->next) {

	pcmk__colocation_t constraint = (pcmk__colocation_t ) cons_iter->data;

	if (constraint->score == INFINITY) {
	member->rsc_cons = g_list_prepend(member->rsc_cons, constraint);
	}
	}
	} else if (!pcmk_is_set(member->flags, pe_rsc_managed)) {
	any_unmanaged = true;
	}
	}

	rsc->rsc_cons = NULL;

	// Treat "R with group" colocations as "R with last member"
	member = group_data->last_child;
	member->rsc_cons_lhs = g_list_concat(member->rsc_cons_lhs,
	rsc->rsc_cons_lhs);
	rsc->rsc_cons_lhs = NULL;
	}

	pe_node_t *
	pcmk__group_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	pe_node_t *node = NULL;
	pe_node_t *group_node = NULL;
	GList *gIter = NULL;
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, rsc);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	if (group_data->first_child == NULL) {
	// Nothing to allocate
	pe__clear_resource_flags(rsc, pe_rsc_provisional);
	return NULL;
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);
	rsc->role = group_data->first_child->role;

	expand_group_colocations(rsc);

	pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, data_set);

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Allocating group %s member %s",
	rsc->id, child_rsc->id);
	node = child_rsc->cmds->allocate(child_rsc, prefer, data_set);
	if (group_node == NULL) {
	group_node = node;
	}
	}

	pe__set_next_role(rsc, group_data->first_child->next_role,
	"first group member");
	pe__clear_resource_flags(rsc, pe_rsc_allocating\|pe_rsc_provisional);

	if (group_data->colocated) {
	return group_node;
	}
	return NULL;
	}

	void group_update_pseudo_status(pe_resource_t * parent, pe_resource_t * child);

	void
	group_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *op = NULL;
	const char *value = NULL;
	GList *gIter = rsc->children;

	pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->create_actions(child_rsc, data_set);
	group_update_pseudo_status(rsc, child_rsc);
	}

	op = start_action(rsc, NULL, TRUE /* !group_data->child_starting */ );
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = custom_action(rsc, started_key(rsc),
	RSC_STARTED, NULL, TRUE /* !group_data->child_starting */ , TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = stop_action(rsc, NULL, TRUE /* !group_data->child_stopping */ );
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = custom_action(rsc, stopped_key(rsc),
	RSC_STOPPED, NULL, TRUE /* !group_data->child_stopping */ , TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROMOTABLE);
	if (crm_is_true(value)) {
	op = custom_action(rsc, demote_key(rsc), RSC_DEMOTE, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = custom_action(rsc, demoted_key(rsc), RSC_DEMOTED, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = custom_action(rsc, promote_key(rsc), RSC_PROMOTE, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);

	op = custom_action(rsc, promoted_key(rsc), RSC_PROMOTED, NULL, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_pseudo\|pe_action_runnable);
	}
	}

	void
	group_update_pseudo_status(pe_resource_t * parent, pe_resource_t * child)
	{
	GList *gIter = child->actions;
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, parent);

	if (group_data->ordered == FALSE) {
	/* If this group is not ordered, then leave the meta-actions as optional */
	return;
	}

	if (group_data->child_stopping && group_data->child_starting) {
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;
	}
	if (pcmk__str_eq(RSC_STOP, action->task, pcmk__str_casei)
	&& pcmk_is_set(action->flags, pe_action_runnable)) {

	group_data->child_stopping = TRUE;
	pe_rsc_trace(action->rsc, "Based on %s the group is stopping", action->uuid);

	} else if (pcmk__str_eq(RSC_START, action->task, pcmk__str_casei)
	&& pcmk_is_set(action->flags, pe_action_runnable)) {
	group_data->child_starting = TRUE;
	pe_rsc_trace(action->rsc, "Based on %s the group is starting", action->uuid);
	}
	}
	}

	void
	group_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = rsc->children;
	pe_resource_t *last_rsc = NULL;
	pe_resource_t *last_active = NULL;
	pe_resource_t *top = uber_parent(rsc);
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, rsc);

	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
	new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set);
	new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	int stop = pe_order_none;
	int stopped = pe_order_implies_then_printed;
	int start = pe_order_implies_then \| pe_order_runnable_left;
	int started =
	pe_order_runnable_left \| pe_order_implies_then \| pe_order_implies_then_printed;

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	if (last_rsc == NULL) {
	if (group_data->ordered) {
	pe__set_order_flags(stop, pe_order_optional);
	stopped = pe_order_implies_then;
	}

	} else if (group_data->colocated) {
	pcmk__new_colocation("group:internal_colocation", NULL, INFINITY,
	child_rsc, last_rsc, NULL, NULL,
	pcmk_is_set(child_rsc->flags, pe_rsc_critical),
	data_set);
	}

	if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
	new_rsc_order(rsc, RSC_DEMOTE, child_rsc, RSC_DEMOTE,
	stop \| pe_order_implies_first_printed, data_set);

	new_rsc_order(child_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stopped, data_set);

	new_rsc_order(child_rsc, RSC_PROMOTE, rsc, RSC_PROMOTED, started, data_set);

	new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
	pe_order_implies_first_printed, data_set);

	}

	order_start_start(rsc, child_rsc, pe_order_implies_first_printed);
	order_stop_stop(rsc, child_rsc, stop \| pe_order_implies_first_printed);

	new_rsc_order(child_rsc, RSC_STOP, rsc, RSC_STOPPED, stopped, data_set);

	new_rsc_order(child_rsc, RSC_START, rsc, RSC_STARTED, started, data_set);

	if (group_data->ordered == FALSE) {
	order_start_start(rsc, child_rsc, start \| pe_order_implies_first_printed);
	if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
	new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
	start \| pe_order_implies_first_printed, data_set);
	}

	} else if (last_rsc != NULL) {
	order_start_start(last_rsc, child_rsc, start);
	order_stop_stop(child_rsc, last_rsc, pe_order_optional \| pe_order_restart);

	if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
	new_rsc_order(last_rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE, start, data_set);
	new_rsc_order(child_rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, pe_order_optional,
	data_set);
	}

	} else {
	order_start_start(rsc, child_rsc, pe_order_none);
	if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
	new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
	pe_order_none, data_set);
	}
	}

	/* Look for partially active groups
	* Make sure they still shut down in sequence
	*/
	if (child_rsc->running_on) {
	if (group_data->ordered
	&& last_rsc
	&& last_rsc->running_on == NULL && last_active && last_active->running_on) {
	order_stop_stop(child_rsc, last_active, pe_order_optional);
	}
	last_active = child_rsc;
	}

	last_rsc = child_rsc;
	}

	if (group_data->ordered && last_rsc != NULL) {
	int stop_stop_flags = pe_order_implies_then;
	int stop_stopped_flags = pe_order_optional;

	order_stop_stop(rsc, last_rsc, stop_stop_flags);
	new_rsc_order(last_rsc, RSC_STOP, rsc, RSC_STOPPED, stop_stopped_flags, data_set);

	if (pcmk_is_set(top->flags, pe_rsc_promotable)) {
	new_rsc_order(rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, stop_stop_flags, data_set);
	new_rsc_order(last_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stop_stopped_flags, data_set);
	}
	}
	}

	void
	group_rsc_colocation_lh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	group_variant_data_t *group_data = NULL;

	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if (rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	gIter = rsc_lh->children;
	pe_rsc_trace(rsc_lh, "Processing constraints from %s", rsc_lh->id);

	get_group_variant_data(group_data, rsc_lh);

	if (group_data->colocated) {
	group_data->first_child->cmds->rsc_colocation_lh(group_data->first_child,
	rsc_rh, constraint,
	data_set);
	return;

	} else if (constraint->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation "
	"between non-colocated group and %s",
	rsc_lh->id, rsc_rh->id);
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_lh(child_rsc, rsc_rh, constraint,
	data_set);
	}
	}

	void
	group_rsc_colocation_rh(pe_resource_t rsc_lh, pe_resource_t rsc_rh,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *gIter = rsc_rh->children;
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, rsc_rh);
	CRM_CHECK(rsc_lh->variant == pe_native, return);

	pe_rsc_trace(rsc_rh, "Processing RH %s of constraint %s (LH is %s)",
	rsc_rh->id, constraint->id, rsc_lh->id);

	if (pcmk_is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return;

	} else if (group_data->colocated && group_data->first_child) {
	if (constraint->score >= INFINITY) {
	/* Ensure RHS is _fully_ up before can start LHS */
	group_data->last_child->cmds->rsc_colocation_rh(rsc_lh,
	group_data->last_child,
	constraint,
	data_set);
	} else {
	/* A partially active RHS is fine */
	group_data->first_child->cmds->rsc_colocation_rh(rsc_lh,
	group_data->first_child,
	constraint,
	data_set);
	}

	return;

	} else if (constraint->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation with"
	" non-colocated group %s", rsc_lh->id, rsc_rh->id);
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint,
	data_set);
	}
	}

	enum pe_action_flags
	group_action_flags(pe_action_t * action, pe_node_t * node)
	{
	GList *gIter = NULL;
	enum pe_action_flags flags = (pe_action_optional \| pe_action_runnable \| pe_action_pseudo);

	for (gIter = action->rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;
	enum action_tasks task = get_complex_task(child, action->task, TRUE);
	const char *task_s = task2text(task);
	pe_action_t *child_action = find_first_action(child->actions, NULL, task_s, node);

	if (child_action) {
	enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);

	if (pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(child_flags, pe_action_optional)) {
	pe_rsc_trace(action->rsc, "%s is mandatory because of %s", action->uuid,
	child_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_optional);
	pe__clear_action_flags(action, pe_action_optional);
	}
	if (!pcmk__str_eq(task_s, action->task, pcmk__str_casei)
	&& pcmk_is_set(flags, pe_action_runnable)
	&& !pcmk_is_set(child_flags, pe_action_runnable)) {

	pe_rsc_trace(action->rsc, "%s is not runnable because of %s", action->uuid,
	child_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_runnable);
	pe__clear_action_flags(action, pe_action_runnable);
	}

	} else if (task != stop_rsc && task != action_demote) {
	pe_rsc_trace(action->rsc, "%s is not runnable because of %s (not found in %s)",
	action->uuid, task_s, child->id);
	pe__clear_raw_action_flags(flags, "group action",
	pe_action_runnable);
	}
	}

	return flags;
	}

	enum pe_graph_flags
	group_update_actions(pe_action_t first, pe_action_t then, pe_node_t *node,
	enum pe_action_flags flags, enum pe_action_flags filter,
	enum pe_ordering type, pe_working_set_t *data_set)
	{
	GList *gIter = then->rsc->children;
	enum pe_graph_flags changed = pe_graph_none;

	CRM_ASSERT(then->rsc != NULL);
	changed \|= native_update_actions(first, then, node, flags, filter, type,
	data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;
	pe_action_t *child_action = find_first_action(child->actions, NULL, then->task, node);

	if (child_action) {
	changed \|= child->cmds->update_actions(first, child_action, node,
	flags, filter, type,
	data_set);
	}
	}

	return changed;
	}

	void
	group_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	GList *gIter = rsc->children;
	GList *saved = constraint->node_list_rh;
	GList *zero = pcmk__copy_node_list(constraint->node_list_rh, true);
	gboolean reset_scores = TRUE;
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, rsc);

	pe_rsc_debug(rsc, "Processing rsc_location %s for %s", constraint->id, rsc->id);

	native_rsc_location(rsc, constraint);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->rsc_location(child_rsc, constraint);
	if (group_data->colocated && reset_scores) {
	reset_scores = FALSE;
	constraint->node_list_rh = zero;
	}
	}

	constraint->node_list_rh = saved;
	g_list_free_full(zero, free);
	}

	void
	group_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	CRM_CHECK(rsc != NULL, return);

	pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);
	native_expand(rsc, data_set);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	GHashTable *
	pcmk__group_merge_weights(pe_resource_t rsc, const char rhs,
	GHashTable nodes, const char attr, float factor,
	uint32_t flags)
	{
	GList *gIter = rsc->rsc_cons_lhs;
	group_variant_data_t *group_data = NULL;

	get_group_variant_data(group_data, rsc);

	if (pcmk_is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "Breaking dependency loop with %s at %s", rsc->id, rhs);
	return nodes;
	}

	pe__set_resource_flags(rsc, pe_rsc_merging);

	nodes =
	group_data->first_child->cmds->merge_weights(group_data->first_child, rhs, nodes, attr,
	factor, flags);

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	nodes = pcmk__native_merge_weights(constraint->rsc_lh, rsc->id, nodes,
	constraint->node_attribute,
	constraint->score / (float) INFINITY,
	flags);
	}

	pe__clear_resource_flags(rsc, pe_rsc_merging);
	return nodes;
	}

	void
	group_append_meta(pe_resource_t * rsc, xmlNode * xml)
	{
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Oct 16, 3:09 PM (23 h, 9 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2530167
Default Alt Text: (379 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions