No OneTemporary
Actions

Size

123 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/common/action_relation_internal.h b/include/crm/common/action_relation_internal.h
	index 9ec5c15179..c0cc38d2e9 100644
	--- a/include/crm/common/action_relation_internal.h
	+++ b/include/crm/common/action_relation_internal.h
	@@ -1,147 +1,151 @@
	/*
	* Copyright 2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H
	# define PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H

	#include <stdint.h> // uint32_t
	#include <crm/common/scheduler_types.h> // pcmk_resource_t, pcmk_action_t

	/*!
	* Flags to indicate the relationship between two actions
	*
	* @COMPAT The values and semantics of these flags should not be changed until
	* the deprecated enum pe_ordering is dropped from the public API.
	*/
	enum pcmk__action_relation_flags {
	//! No relation (compare with equality rather than bit set)
	pcmk__ar_none = 0U,

	//! Actions are ordered (optionally, if no other flags are set)
	pcmk__ar_ordered = (1U << 0),

	//! Relation applies only if 'first' cannot be part of a live migration
	pcmk__ar_if_first_unmigratable = (1U << 1),

	/*!
	* If 'then' is required, 'first' becomes required (and becomes unmigratable
	* if 'then' is); also, if 'first' is a stop of a blocked resource, 'then'
	* becomes unrunnable
	*/
	pcmk__ar_then_implies_first = (1U << 4),

	/*!
	* If 'first' is required, 'then' becomes required; if 'first' is a stop of
	* a blocked resource, 'then' becomes unrunnable
	*/
	pcmk__ar_first_implies_then = (1U << 5),

	/*!
	* If 'then' is required and for a promoted instance, 'first' becomes
	* required (and becomes unmigratable if 'then' is)
	*/
	pcmk__ar_promoted_then_implies_first = (1U << 6),

	/*!
	* 'first' is runnable only if 'then' is both runnable and migratable,
	* and 'first' becomes required if 'then' is
	*/
	pcmk__ar_unmigratable_then_blocks = (1U << 7),

	//! 'then' is runnable (and migratable) only if 'first' is runnable
	pcmk__ar_unrunnable_first_blocks = (1U << 8),

	//! If 'first' is unrunnable, 'then' becomes a real, unmigratable action
	pcmk__ar_first_else_then = (1U << 9),

	//! If 'first' is required, 'then' action for instance on same node is
	pcmk__ar_first_implies_same_node_then = (1U << 10),

	/*!
	* Disable relation if 'first' is unrunnable and for an active resource,
	* otherwise order actions and make 'then' unrunnable if 'first' is.
	*
	* This is used to order a bundle replica's start of its container before a
	* probe of its remote connection resource, in case the connection uses the
	* REMOTE_CONTAINER_HACK to replace the connection address with where the
	* container is running.
	*/
	pcmk__ar_nested_remote_probe = (1U << 11),

	/*!
	* If 'first' is for a blocked resource, make 'then' unrunnable.
	*
	* If 'then' is required, make 'first' required, make 'first' unmigratable
	* if 'then' is unmigratable, and make 'then' unrunnable if 'first' is
	* unrunnable.
	*
	* If 'then' is unrunnable and for the same resource as 'first', make
	* 'first' required if it is runnable, and make 'first' unmigratable if
	* 'then' is unmigratable.
	*
	* This is used for "stop then start primitive" (restarts) and
	* "stop group member then stop previous member".
	*/
	pcmk__ar_intermediate_stop = (1U << 12),

	/*!
	* The actions must be serialized if in the same transition but can be in
	* either order. (In practice, we always arrange them as 'first' then
	* 'then', so they end up being essentially the same as optional orderings.)
	*
	* @TODO Handle more intelligently -- for example, we could schedule the
	* action with the fewest inputs first, so we're more likely to execute at
	* least one if there is a failure during the transition. Or, we could
	* prefer certain action types over others, or base it on resource priority.
	*/
	pcmk__ar_serialize = (1U << 14),

	//! Relation applies only if actions are on same node
	pcmk__ar_if_on_same_node = (1U << 15),

	//! If 'then' is required, 'first' must be added to the transition graph
	pcmk__ar_then_implies_first_graphed = (1U << 16),

	//! If 'first' is required and runnable, 'then' must be in graph
	pcmk__ar_first_implies_then_graphed = (1U << 17),

	//! User-configured asymmetric ordering
	pcmk__ar_asymmetric = (1U << 20),

	//! Actions are ordered if on same node (or migration target for migrate_to)
	pcmk__ar_if_on_same_node_or_target = (1U << 21),

	//! 'then' action is runnable if certain number of 'first' instances are
	pcmk__ar_min_runnable = (1U << 22),

	//! Ordering applies only if 'first' is required and on same node as 'then'
	pcmk__ar_if_required_on_same_node = (1U << 23),

	//! Ordering applies even if 'first' runs on guest node created by 'then'
	pcmk__ar_guest_allowed = (1U << 24),

	//! If 'then' action becomes required, 'first' becomes optional
	pcmk__ar_then_cancels_first = (1U << 25),
	};

	-// Action relation object
	+/* Action relation object
	+ *
	+ * The most common type of relation is an ordering, in which case action1 etc.
	+ * refers to the "first" action, and action2 etc. refers to the "then" action.
	+ */
	typedef struct {
	int id; // Counter to identify relation
	uint32_t flags; // Group of enum pcmk__action_relation_flags
	pcmk_resource_t *rsc1; // Resource for first action, if any
	pcmk_action_t *action1; // First action in relation
	char *task1; // Action name or key for first action
	pcmk_resource_t *rsc2; // Resource for second action, if any
	pcmk_action_t *action2; // Second action in relation
	- char *rh_action_task; // Action name or key for 'then' action
	+ char *task2; // Action name or key for second action
	} pcmk__action_relation_t;

	typedef struct pe_action_wrapper_s pcmk__related_action_t;

	#endif // PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H
	diff --git a/lib/pacemaker/pcmk_sched_migration.c b/lib/pacemaker/pcmk_sched_migration.c
	index 317b791bc1..0e13be79a9 100644
	--- a/lib/pacemaker/pcmk_sched_migration.c
	+++ b/lib/pacemaker/pcmk_sched_migration.c
	@@ -1,404 +1,404 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add migration source and target meta-attributes to an action
	*
	* \param[in,out] action Action to add meta-attributes to
	* \param[in] source Node to add as migration source
	* \param[in] target Node to add as migration target
	*/
	static void
	add_migration_meta(pcmk_action_t action, const pcmk_node_t source,
	const pcmk_node_t *target)
	{
	add_hash_param(action->meta, XML_LRM_ATTR_MIGRATE_SOURCE,
	source->details->uname);

	add_hash_param(action->meta, XML_LRM_ATTR_MIGRATE_TARGET,
	target->details->uname);
	}

	/*!
	* \internal
	* \brief Create internal migration actions for a migrateable resource
	*
	* \param[in,out] rsc Resource to create migration actions for
	* \param[in] current Node that resource is originally active on
	*/
	void
	pcmk__create_migration_actions(pcmk_resource_t rsc, const pcmk_node_t current)
	{
	pcmk_action_t *migrate_to = NULL;
	pcmk_action_t *migrate_from = NULL;
	pcmk_action_t *start = NULL;
	pcmk_action_t *stop = NULL;

	pe_rsc_trace(rsc, "Creating actions to %smigrate %s from %s to %s",
	((rsc->partial_migration_target == NULL)? "" : "partially "),
	rsc->id, pe__node_name(current),
	pe__node_name(rsc->allocated_to));
	start = start_action(rsc, rsc->allocated_to, TRUE);
	stop = stop_action(rsc, current, TRUE);

	if (rsc->partial_migration_target == NULL) {
	migrate_to = custom_action(rsc, pcmk__op_key(rsc->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	PCMK_ACTION_MIGRATE_TO, current, TRUE,
	rsc->cluster);
	}
	migrate_from = custom_action(rsc, pcmk__op_key(rsc->id,
	PCMK_ACTION_MIGRATE_FROM, 0),
	PCMK_ACTION_MIGRATE_FROM, rsc->allocated_to,
	TRUE, rsc->cluster);

	pe__set_action_flags(start, pcmk_action_migratable);
	pe__set_action_flags(stop, pcmk_action_migratable);

	// This is easier than trying to delete it from the graph
	pe__set_action_flags(start, pcmk_action_pseudo);

	if (rsc->partial_migration_target == NULL) {
	pe__set_action_flags(migrate_from, pcmk_action_migratable);
	pe__set_action_flags(migrate_to, pcmk_action_migratable);
	migrate_to->needs = start->needs;

	// Probe -> migrate_to -> migrate_from
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_TO, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_TO, 0),
	NULL,
	rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_FROM, 0),
	NULL,
	pcmk__ar_ordered\|pcmk__ar_unmigratable_then_blocks,
	rsc->cluster);
	} else {
	pe__set_action_flags(migrate_from, pcmk_action_migratable);
	migrate_from->needs = start->needs;

	// Probe -> migrate_from (migrate_to already completed)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_FROM, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	}

	// migrate_from before stop or start
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_FROM, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	pcmk__ar_ordered\|pcmk__ar_unmigratable_then_blocks,
	rsc->cluster);
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_FROM, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered
	\|pcmk__ar_unmigratable_then_blocks
	\|pcmk__ar_first_else_then,
	rsc->cluster);

	if (migrate_to != NULL) {
	add_migration_meta(migrate_to, current, rsc->allocated_to);

	if (!rsc->is_remote_node) {
	/* migrate_to takes place on the source node, but can affect the
	* target node depending on how the agent is written. Because of
	* this, pending migrate_to actions must be recorded in the CIB,
	* in case the source node loses membership while the migrate_to
	* action is still in flight.
	*
	* However we know Pacemaker Remote connection resources don't
	* require this, so we skip this for them. (Although it wouldn't
	* hurt, and now that record-pending defaults to true, skipping it
	* matters even less.)
	*/
	add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
	}
	}

	add_migration_meta(migrate_from, current, rsc->allocated_to);
	}

	/*!
	* \internal
	* \brief Abort a dangling migration by scheduling a stop (and possibly cleanup)
	*
	* \param[in] data Source node of dangling migration
	* \param[in,out] user_data Resource involved in dangling migration
	*/
	void
	pcmk__abort_dangling_migration(void data, void user_data)
	{
	const pcmk_node_t dangling_source = (const pcmk_node_t ) data;
	pcmk_resource_t rsc = (pcmk_resource_t ) user_data;

	pcmk_action_t *stop = NULL;
	bool cleanup = pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_remove_after_stop);

	pe_rsc_trace(rsc,
	"Scheduling stop%s for %s on %s due to dangling migration",
	(cleanup? " and cleanup" : ""), rsc->id,
	pe__node_name(dangling_source));
	stop = stop_action(rsc, dangling_source, FALSE);
	pe__set_action_flags(stop, pcmk_action_migration_abort);
	if (cleanup) {
	pcmk__schedule_cleanup(rsc, dangling_source, false);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource can migrate
	*
	* \param[in] rsc Resource to check
	* \param[in] node Resource's current node
	*
	* \return true if \p rsc can migrate, otherwise false
	*/
	bool
	pcmk__rsc_can_migrate(const pcmk_resource_t rsc, const pcmk_node_t current)
	{
	CRM_CHECK(rsc != NULL, return false);

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_migratable)) {
	pe_rsc_trace(rsc, "%s cannot migrate because "
	"the configuration does not allow it",
	rsc->id);
	return false;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc, "%s cannot migrate because it is not managed",
	rsc->id);
	return false;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pe_rsc_trace(rsc, "%s cannot migrate because it is failed",
	rsc->id);
	return false;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	pe_rsc_trace(rsc, "%s cannot migrate because it has a start pending",
	rsc->id);
	return false;
	}

	if ((current == NULL) \|\| current->details->unclean) {
	pe_rsc_trace(rsc, "%s cannot migrate because "
	"its current node (%s) is unclean",
	rsc->id, pe__node_name(current));
	return false;
	}

	if ((rsc->allocated_to == NULL) \|\| rsc->allocated_to->details->unclean) {
	pe_rsc_trace(rsc, "%s cannot migrate because "
	"its next node (%s) is unclean",
	rsc->id, pe__node_name(rsc->allocated_to));
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Get an action name from an action or operation key
	*
	* \param[in] action If not NULL, get action name from here
	* \param[in] key If not NULL, get action name from here
	*
	* \return Newly allocated copy of action name (or NULL if none available)
	*/
	static char *
	task_from_action_or_key(const pcmk_action_t action, const char key)
	{
	char *res = NULL;

	if (action != NULL) {
	res = strdup(action->task);
	CRM_ASSERT(res != NULL);
	} else if (key != NULL) {
	parse_op_key(key, NULL, &res, NULL);
	}
	return res;
	}

	/*!
	* \internal
	* \brief Order migration actions equivalent to a given ordering
	*
	* Orderings involving start, stop, demote, and promote actions must be honored
	* during a migration as well, so duplicate any such ordering for the
	* corresponding migration actions.
	*
	* \param[in,out] order Ordering constraint to check
	*/
	void
	pcmk__order_migration_equivalents(pcmk__action_relation_t *order)
	{
	char *first_task = NULL;
	char *then_task = NULL;
	bool then_migratable;
	bool first_migratable;

	// Only orderings between unrelated resources are relevant
	if ((order->rsc1 == NULL) \|\| (order->rsc2 == NULL)
	\|\| (order->rsc1 == order->rsc2)
	\|\| is_parent(order->rsc1, order->rsc2)
	\|\| is_parent(order->rsc2, order->rsc1)) {
	return;
	}

	// Only orderings involving at least one migratable resource are relevant
	first_migratable = pcmk_is_set(order->rsc1->flags, pcmk_rsc_migratable);
	then_migratable = pcmk_is_set(order->rsc2->flags, pcmk_rsc_migratable);
	if (!first_migratable && !then_migratable) {
	return;
	}

	// Check which actions are involved
	first_task = task_from_action_or_key(order->action1, order->task1);
	- then_task = task_from_action_or_key(order->action2, order->rh_action_task);
	+ then_task = task_from_action_or_key(order->action2, order->task2);

	if (pcmk__str_eq(first_task, PCMK_ACTION_START, pcmk__str_none)
	&& pcmk__str_eq(then_task, PCMK_ACTION_START, pcmk__str_none)) {

	uint32_t flags = pcmk__ar_ordered;

	if (first_migratable && then_migratable) {
	/* A start then B start
	* -> A migrate_from then B migrate_to */
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id,
	PCMK_ACTION_MIGRATE_FROM, 0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	NULL, flags, order->rsc1->cluster);
	}

	if (then_migratable) {
	if (first_migratable) {
	pe__set_order_flags(flags, pcmk__ar_if_first_unmigratable);
	}

	/* A start then B start
	* -> A start then B migrate_to (if start is not part of a
	* migration)
	*/
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id,
	PCMK_ACTION_START, 0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	NULL, flags, order->rsc1->cluster);
	}

	} else if (then_migratable
	&& pcmk__str_eq(first_task, PCMK_ACTION_STOP, pcmk__str_none)
	&& pcmk__str_eq(then_task, PCMK_ACTION_STOP, pcmk__str_none)) {

	uint32_t flags = pcmk__ar_ordered;

	if (first_migratable) {
	pe__set_order_flags(flags, pcmk__ar_if_first_unmigratable);
	}

	/* For an ordering "stop A then stop B", if A is moving via restart, and
	* B is migrating, enforce that B's migrate_to occurs after A's stop.
	*/
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id, PCMK_ACTION_STOP, 0),
	NULL,
	order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	NULL, flags, order->rsc1->cluster);

	// Also order B's migrate_from after A's stop during partial migrations
	if (order->rsc2->partial_migration_target != NULL) {
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id, PCMK_ACTION_STOP,
	0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_FROM, 0),
	NULL, flags, order->rsc1->cluster);
	}

	} else if (pcmk__str_eq(first_task, PCMK_ACTION_PROMOTE, pcmk__str_none)
	&& pcmk__str_eq(then_task, PCMK_ACTION_START, pcmk__str_none)) {

	uint32_t flags = pcmk__ar_ordered;

	if (then_migratable) {
	/* A promote then B start
	* -> A promote then B migrate_to */
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id,
	PCMK_ACTION_PROMOTE, 0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	NULL, flags, order->rsc1->cluster);
	}

	} else if (pcmk__str_eq(first_task, PCMK_ACTION_DEMOTE, pcmk__str_none)
	&& pcmk__str_eq(then_task, PCMK_ACTION_STOP, pcmk__str_none)) {

	uint32_t flags = pcmk__ar_ordered;

	if (then_migratable) {
	/* A demote then B stop
	* -> A demote then B migrate_to */
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id,
	PCMK_ACTION_DEMOTE, 0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_TO, 0),
	NULL, flags, order->rsc1->cluster);

	// Order B migrate_from after A demote during partial migrations
	if (order->rsc2->partial_migration_target != NULL) {
	pcmk__new_ordering(order->rsc1,
	pcmk__op_key(order->rsc1->id,
	PCMK_ACTION_DEMOTE, 0),
	NULL, order->rsc2,
	pcmk__op_key(order->rsc2->id,
	PCMK_ACTION_MIGRATE_FROM, 0),
	NULL, flags, order->rsc1->cluster);
	}
	}
	}

	free(first_task);
	free(then_task);
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index bff6fde153..0044040130 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1501 +1,1501 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(scheduler->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	crm_warn("Unknown action '%s' specified in order constraint", action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pcmk__wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pcmk__ar_none; // so we trace-log all flags set

	switch (kind) {
	case pe_order_kind_optional:
	pe__set_order_flags(flags, pcmk__ar_ordered);
	break;

	case pe_order_kind_serialize:
	/* This flag is not used anywhere directly but means the relation
	* will not match an equality comparison against pcmk__ar_none or
	* pcmk__ar_ordered.
	*/
	pe__set_order_flags(flags, pcmk__ar_serialize);
	break;

	case pe_order_kind_mandatory:
	pe__set_order_flags(flags, pcmk__ar_ordered);
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pcmk__ar_asymmetric);
	break;

	case ordering_symmetric:
	pe__set_order_flags(flags, pcmk__ar_first_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pe__set_order_flags(flags,
	pcmk__ar_unrunnable_first_blocks);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pcmk__ar_then_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] scheduler Scheduler data
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pcmk_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pcmk_scheduler_t *scheduler)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(scheduler->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pe_warn_once(pcmk__wo_order_inst,
	"Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
	XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pcmk_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pe_warn_once(pcmk__wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pcmk_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pcmk_action_min_runnable);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	rsc_first->cluster);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	rsc_first->cluster);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *rsc_then = NULL;
	pcmk_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pcmk__ar_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	scheduler);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	scheduler);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Group of enum pcmk__action_relation_flags
	* \param[in,out] sched Scheduler data to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pcmk_resource_t first_rsc, char first_action_task,
	pcmk_action_t first_action, pcmk_resource_t then_rsc,
	char then_action_task, pcmk_action_t then_action,
	uint32_t flags, pcmk_scheduler_t *sched)
	{
	pcmk__action_relation_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pcmk__action_relation_t));
	CRM_ASSERT(order != NULL);

	order->id = sched->order_id++;
	order->flags = flags;
	order->rsc1 = first_rsc;
	order->rsc2 = then_rsc;
	order->action1 = first_action;
	order->action2 = then_action;
	order->task1 = first_action_task;
	- order->rh_action_task = then_action_task;
	+ order->task2 = then_action_task;

	if ((order->task1 == NULL) && (first_action != NULL)) {
	order->task1 = strdup(first_action->uuid);
	}

	- if ((order->rh_action_task == NULL) && (then_action != NULL)) {
	- order->rh_action_task = strdup(then_action->uuid);
	+ if ((order->task2 == NULL) && (then_action != NULL)) {
	+ order->task2 = strdup(then_action->uuid);
	}

	if ((order->rsc1 == NULL) && (first_action != NULL)) {
	order->rsc1 = first_action->rsc;
	}

	if ((order->rsc2 == NULL) && (then_action != NULL)) {
	order->rsc2 = then_action->rsc;
	}

	pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->order_id - 1),
	pcmk__s(order->task1, "an underspecified action"),
	- pcmk__s(order->rh_action_task, "an underspecified action"));
	+ pcmk__s(order->task2, "an underspecified action"));

	sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pcmk_scheduler_t scheduler)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pcmk_resource_t *last = NULL;
	pcmk_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pcmk__ar_ordered;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pcmk_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, scheduler);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] scheduler Scheduler data
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pcmk__ar_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
	pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);

	free(task);
	pe__set_action_flags(unordered_action, pcmk_action_min_runnable);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	scheduler);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	scheduler);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2,
	XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pcmk_scheduler_t *scheduler)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pcmk_resource_t *rsc_first = NULL;
	pcmk_resource_t *rsc_then = NULL;
	pcmk_tag_t *tag_first = NULL;
	pcmk_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
	&tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, scheduler,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, scheduler,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, scheduler);
	}
	}

	static bool
	ordering_is_invalid(pcmk_action_t action, pcmk__related_action_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pcmk__ar_guest_allowed)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if (((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target)
	&& (action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	pcmk__related_action_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = (enum pe_ordering) pcmk__ar_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pcmk_node_t node, pcmk_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t action = (pcmk_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if (!pe__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk_rsc_managed\|pcmk_rsc_blocked)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pe__node_name(node));
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pcmk_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pcmk_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->actions, key, NULL);
	free(key);
	free(task);
	} else {
	crm_err("Invalid operation key (bug?): %s", original_key);
	}
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pcmk_action_t *first_action,
	const pcmk_resource_t *rsc,
	pcmk__action_relation_t *order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pcmk__ar_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->action2 != NULL) {
	then_actions = g_list_prepend(NULL, order->action2);

	} else {
	- then_actions = find_actions_by_task(rsc, order->rh_action_task);
	+ then_actions = find_actions_by_task(rsc, order->task2);
	}

	if (then_actions == NULL) {
	pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	- order->id, order->rh_action_task, rsc->id);
	+ order->id, order->task2, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {

	pe_rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	- first_action->uuid, order->rh_action_task, rsc->id);
	+ first_action->uuid, order->task2, rsc->id);
	pe__clear_order_flags(flags, pcmk__ar_first_implies_then);
	}

	if ((first_action == NULL)
	&& !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {

	pe_rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then_action_iter = (pcmk_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pe__clear_action_flags(then_action_iter, pcmk_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->task1, order->rsc1->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pcmk_resource_t first_rsc, pcmk__action_relation_t order)
	{
	GList *first_actions = NULL;
	pcmk_action_t *first_action = order->action1;
	pcmk_resource_t *then_rsc = order->rsc2;

	CRM_ASSERT(first_rsc != NULL);
	pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->task1);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->task1, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc,
	TRUE) == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else {
	pe_rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->task1, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	first_rsc->cluster);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->action2 == NULL) {
	pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->action2->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->action2, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pcmk_action_t *) data,
	(pcmk_scheduler_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Scheduler data
	*/
	void
	pcmk__apply_orderings(pcmk_scheduler_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);

	for (GList *iter = sched->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	pcmk_resource_t *rsc = order->rsc1;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rsc2;
	if (rsc != NULL) {
	order_resource_actions_after(order->action1, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->action1, order->action2, order->flags);
	}
	}

	g_list_foreach(sched->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->actions));
	g_list_foreach(sched->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pcmk_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_action_t before = (pcmk_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pe__node_name(before->node),
	after_desc, pe__node_name(after->node));
	order_actions(before, after, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_ordered);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index 6cc10a3002..30f1443ce8 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,903 +1,903 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pcmk_action_t probe, const pcmk_resource_t rsc,
	const pcmk_node_t *node)
	{
	// Check whether resource is currently active on node
	pcmk_node_t *running = pe_find_node_id(rsc->running_on, node->details->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pcmk_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pcmk_resource_t rsc1, pcmk_resource_t rsc2)
	{
	if ((rsc1->allocated_to != NULL)
	&& (g_hash_table_lookup(rsc1->known_on,
	rsc1->allocated_to->details->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pcmk_node_t *node)
	{
	const pcmk_resource_t *guest_rsc = node->details->remote_rsc->container;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk_rsc_failed)
	\|\| (guest_rsc->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->role > pcmk_role_stopped)
	&& (guest_rsc->allocated_to != NULL)
	&& (pe_find_node(guest_rsc->running_on,
	guest_rsc->allocated_to->details->uname) == NULL));
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pcmk_action_t *
	probe_action(pcmk_resource_t rsc, pcmk_node_t node)
	{
	pcmk_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	role2text(rsc->role), rsc->id, pe__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE,
	rsc->cluster);
	pe__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pcmk_action_t *probe = NULL;
	pcmk_node_t *allowed = NULL;
	pcmk_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(rsc->cluster, rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (rsc->is_remote_node) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->children != NULL) {
	return pcmk__probe_resource_list(rsc->children, node);
	}

	if ((rsc->container != NULL) && !rsc->is_remote_node) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	} else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	if (allowed->rsc_discover_mode == pcmk_probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pe__is_guest_node(node)) {
	pcmk_resource_t *guest = node->details->remote_rsc->container;

	if (guest->role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pe_rsc_is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->running_on == NULL)) {
	pe__set_order_flags(flags, pcmk__ar_unrunnable_first_blocks);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->cluster);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->cluster);

	return true;

	no_probe:
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->details->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pcmk_action_t *probe,
	const pcmk_action_t *then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& pe__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta, "stonith_action");

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	&& !pe__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_probe_orderings_for_stops(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pcmk_action_t *first = NULL;
	pcmk_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->rsc1 == NULL) \|\| (order->rsc1 == order->rsc2)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->action1;
	then = order->action2;
	if (((first == NULL) && (order->task1 == NULL))
	- \|\| ((then == NULL) && (order->rh_action_task == NULL))) {
	+ \|\| ((then == NULL) && (order->task2 == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->task1,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* container. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the container starts again.
	*/
	if ((order->rsc2 != NULL) && (order->rsc1->container == order->rsc2)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	- && pcmk__ends_with(order->rh_action_task,
	+ && pcmk__ends_with(order->task2,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pe__set_order_flags(order_flags, pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pcmk__ar_if_on_same_node)) {
	pe__set_order_flags(order_flags, pcmk__ar_if_on_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pcmk__ar_if_required_on_same_node)
	\|\| (order->flags == pcmk__ar_if_on_same_node_or_target)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->rsc1, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rsc2 != NULL) {
	- then_actions = find_actions(order->rsc2->actions,
	- order->rh_action_task, NULL);
	+ then_actions = find_actions(order->rsc2->actions, order->task2,
	+ NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->task1 : first->uuid),
	- ((then == NULL)? order->rh_action_task : then->uuid),
	+ ((then == NULL)? order->task2 : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk_action_t then = (pcmk_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pcmk_action_t *probe,
	pcmk__related_action_t *after)
	{
	uint32_t flags = pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pcmk__ar_unrunnable_first_blocks` order type for that
	* as long as any of the clone instances are running to prevent them from
	* being unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->variant <= pcmk_rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pcmk__ar_unrunnable_first_blocks)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->action->uuid, pe__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	if (then->action->rsc->running_on
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pe__node_name(probe->node),
	then->action->uuid, pe__node_name(then->action->node), flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pcmk_action_t probe, pcmk_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pcmk_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| (probe->rsc == NULL)
	\|\| (probe->rsc->variant != pcmk_rsc_variant_primitive)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pe__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->uuid, pe__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant == pcmk_rsc_variant_primitive)
	&& (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then = (pcmk_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant > pcmk_rsc_variant_group)) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pcmk__related_action_t *after_wrapper = iter->data;

	/* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pcmk__ar_first_implies_same_node_then though. It's now
	* only used for unfencing case, which tends to introduce transition
	* loops...
	*/
	if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pcmk__ar_then_implies_first_graphed
	* \|pcmk__ar_unrunnable_first_blocks
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->variant < pcmk_rsc_variant_group)
	\|\| (probe->rsc->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)
	\|\| (after_wrapper->action->rsc->variant > pcmk_rsc_variant_group)
	\|\| (after->rsc != after_wrapper->action->rsc->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->variant > pcmk_rsc_variant_group) && interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != after_wrapper->action->rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pe__node_name(after->node),
	after_wrapper->action->uuid,
	pe__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	clear_actions_tracking_flag(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	pe__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, add_start_restart_orderings_for_rsc,
	NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pcmk_action_t probe = (pcmk_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->cluster);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pcmk_scheduler_t *scheduler)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pcmk__related_action_t *before = actions->data;

	pcmk_action_t *first = before->action;
	pcmk_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pe_rsc_is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Applying %s before %s %d", first->uuid, start->uuid,
	pe__const_top_resource(first_rsc, false)->variant);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pcmk_scheduler_t *scheduler)
	{
	// Add orderings for "probe then X"
	g_list_foreach(scheduler->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(scheduler);

	order_then_probes(scheduler);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pcmk_scheduler_t *scheduler)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	const char *probed = NULL;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(scheduler, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) { // ... or nodes that need fencing
	continue;

	} else if (!node->details->rsc_discovery_enabled) {
	// The user requested that probes not be done on this node
	continue;
	}

	/* This is no longer needed for live clusters, since the probe_complete
	* node attribute will never be in the CIB. However this is still useful
	* for processing old saved CIBs (< 1.1.14), including the
	* reprobe-target_rc regression test.
	*/
	probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pcmk_action_t *probe_op = NULL;

	probe_op = custom_action(NULL,
	crm_strdup_printf("%s-%s", CRM_OP_REPROBE,
	node->details->uname),
	CRM_OP_REPROBE, node, FALSE, scheduler);
	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT,
	XML_BOOLEAN_TRUE);
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(scheduler->resources, node);
	}
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index f531611555..ae1fbce361 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,483 +1,483 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler != NULL) {
	set_working_set_defaults(scheduler);
	}
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	scheduler->priv = NULL;
	free(scheduler);
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->failed != NULL) {
	free_xml(scheduler->failed);
	}
	scheduler->failed = create_xml_node(NULL, "failed-ops");

	if (scheduler->now == NULL) {
	scheduler->now = crm_time_new(NULL);
	}

	if (scheduler->dc_uuid == NULL) {
	scheduler->dc_uuid = crm_element_value_copy(scheduler->input,
	XML_ATTR_DC_UUID);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, XML_ATTR_HAVE_QUORUM)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_quorate);
	} else {
	pe__clear_working_set_flags(scheduler, pcmk_sched_quorate);
	}

	scheduler->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
	scheduler->input, LOG_NEVER);
	scheduler->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
	scheduler->input, LOG_NEVER);

	section = get_xpath_object("//" XML_CIB_TAG_CRMCONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_location_only\|pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	crm_warn("Fencing and resource management disabled due to lack of quorum");
	}

	section = get_xpath_object("//" XML_CIB_TAG_NODES, scheduler->input,
	LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_TAGS, scheduler->input,
	LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	section = get_xpath_object("//"XML_CIB_TAG_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_no_counts)) {
	for (GList *item = scheduler->resources; item != NULL;
	item = item->next) {
	((pcmk_resource_t *) (item->data))->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	pe__set_working_set_flags(scheduler, pcmk_sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->details->uname for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
	"(guest or remote)" : pe__node_name(node)));

	if (node->details->attrs != NULL) {
	g_hash_table_destroy(node->details->attrs);
	}
	if (node->details->utilization != NULL) {
	g_hash_table_destroy(node->details->utilization);
	}
	if (node->details->digest_cache != NULL) {
	g_hash_table_destroy(node->details->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->details->allocated_rsc);
	free(node->details);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__action_relation_t *order = iterator->data;

	iterator = iterator->next;

	free(order->task1);
	- free(order->rh_action_task);
	+ free(order->task2);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	pe__clear_working_set_flags(scheduler, pcmk_sched_have_status);
	if (scheduler->config_hash != NULL) {
	g_hash_table_destroy(scheduler->config_hash);
	}

	if (scheduler->singletons != NULL) {
	g_hash_table_destroy(scheduler->singletons);
	}

	if (scheduler->tickets) {
	g_hash_table_destroy(scheduler->tickets);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	free(scheduler->dc_uuid);

	crm_trace("deleting resources");
	pe_free_resources(scheduler->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	free_xml(scheduler->graph);
	crm_time_free(scheduler->now);
	free_xml(scheduler->input);
	free_xml(scheduler->failed);

	set_working_set_defaults(scheduler);

	CRM_CHECK(scheduler->ordering_constraints == NULL,;
	);
	CRM_CHECK(scheduler->placement_constraints == NULL,;
	);
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->ordering_constraints));
	pe__free_ordering(scheduler->ordering_constraints);
	scheduler->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->placement_constraints));
	pe__free_location(scheduler->placement_constraints);
	scheduler->placement_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->colocation_constraints));
	g_list_free_full(scheduler->colocation_constraints, free);
	scheduler->colocation_constraints = NULL;

	crm_trace("Deleting %d ticket constraints",
	g_list_length(scheduler->ticket_constraints));
	g_list_free_full(scheduler->ticket_constraints, free);
	scheduler->ticket_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	void *priv = scheduler->priv;

	memset(scheduler, 0, sizeof(pcmk_scheduler_t));

	scheduler->priv = priv;
	scheduler->order_id = 1;
	scheduler->action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;

	scheduler->flags = 0x0ULL;

	pe__set_working_set_flags(scheduler,
	pcmk_sched_symmetric_cluster
	\|pcmk_sched_stop_removed_resources
	\|pcmk_sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, "true")) {
	pe__set_working_set_flags(scheduler, pcmk_sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;

	pcmk_resource_t *match =
	parent->fns->find_rsc(parent, id, NULL, flags);
	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pe_find_node(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->details->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (pcmk__str_eq(node->details->uname, node_name, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 8:06 PM (6 h, 9 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665650
Default Alt Text: (123 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions