No OneTemporary
Actions

Size

626 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/common/action_relation_internal.h b/include/crm/common/action_relation_internal.h
	index 9cb3362367..2503501844 100644
	--- a/include/crm/common/action_relation_internal.h
	+++ b/include/crm/common/action_relation_internal.h
	@@ -1,37 +1,43 @@
	/*
	* Copyright 2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H
	# define PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H

	/*!
	* Flags to indicate the relationship between two actions
	*
	* @COMPAT The values and semantics of these flags should not be changed until
	* the deprecated enum pe_ordering is dropped from the public API.
	*/
	enum pcmk__action_relation_flags {
	//! No relation (compare with equality rather than bit set)
	pcmk__ar_none = 0U,

	//! Actions are ordered (optionally, if no other flags are set)
	pcmk__ar_ordered = (1U << 0),

	//! Relation applies only if 'first' cannot be part of a live migration
	pcmk__ar_if_first_unmigratable = (1U << 1),

	/*!
	* If 'then' is required, 'first' becomes required (and becomes unmigratable
	* if 'then' is); also, if 'first' is a stop of a blocked resource, 'then'
	* becomes unrunnable
	*/
	pcmk__ar_then_implies_first = (1U << 4),
	+
	+ /*!
	+ * If 'first' is required, 'then' becomes required; if 'first' is a stop of
	+ * a blocked resource, 'then' becomes unrunnable
	+ */
	+ pcmk__ar_first_implies_then = (1U << 5),
	};

	#endif // PCMK__CRM_COMMON_ACTION_RELATION_INTERNAL__H
	diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
	index 63e58ec296..8d5626a8db 100644
	--- a/lib/pacemaker/pcmk_sched_actions.c
	+++ b/lib/pacemaker/pcmk_sched_actions.c
	@@ -1,1927 +1,1928 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/param.h>
	#include <glib.h>

	#include <crm/lrmd_internal.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get the action flags relevant to ordering constraints
	*
	* \param[in,out] action Action to check
	* \param[in] node Node that other action in the ordering is on
	* (used only for clone resource actions)
	*
	* \return Action flags that should be used for orderings
	*/
	static uint32_t
	action_flags_for_ordering(pe_action_t action, const pe_node_t node)
	{
	bool runnable = false;
	uint32_t flags;

	// For non-resource actions, return the action flags
	if (action->rsc == NULL) {
	return action->flags;
	}

	/* For non-clone resources, or a clone action not assigned to a node,
	* return the flags as determined by the resource method without a node
	* specified.
	*/
	flags = action->rsc->cmds->action_flags(action, NULL);
	if ((node == NULL) \|\| !pe_rsc_is_clone(action->rsc)) {
	return flags;
	}

	/* Otherwise (i.e., for clone resource actions on a specific node), first
	* remember whether the non-node-specific action is runnable.
	*/
	runnable = pcmk_is_set(flags, pcmk_action_runnable);

	// Then recheck the resource method with the node
	flags = action->rsc->cmds->action_flags(action, node);

	/* For clones in ordering constraints, the node-specific "runnable" doesn't
	* matter, just the non-node-specific setting (i.e., is the action runnable
	* anywhere).
	*
	* This applies only to runnable, and only for ordering constraints. This
	* function shouldn't be used for other types of constraints without
	* changes. Not very satisfying, but it's logical and appears to work well.
	*/
	if (runnable && !pcmk_is_set(flags, pcmk_action_runnable)) {
	pe__set_raw_action_flags(flags, action->rsc->id, pcmk_action_runnable);
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Get action UUID that should be used with a resource ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the UUID and resource of the first action in an
	* ordering, this returns the UUID of the action that should actually be used
	* for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
	*
	* \param[in] first_uuid UUID of first action in ordering
	* \param[in] first_rsc Resource of first action in ordering
	*
	* \return Newly allocated copy of UUID to use with ordering
	* \note It is the caller's responsibility to free the return value.
	*/
	static char *
	action_uuid_for_ordering(const char first_uuid, const pe_resource_t first_rsc)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *first_task_str = NULL;
	enum action_tasks first_task = pcmk_action_unspecified;
	enum action_tasks remapped_task = pcmk_action_unspecified;

	// Only non-notify actions for collective resources need remapping
	if ((strstr(first_uuid, PCMK_ACTION_NOTIFY) != NULL)
	\|\| (first_rsc->variant < pcmk_rsc_variant_group)) {
	goto done;
	}

	// Only non-recurring actions need remapping
	CRM_ASSERT(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
	if (interval_ms > 0) {
	goto done;
	}

	first_task = text2task(first_task_str);
	switch (first_task) {
	case pcmk_action_stop:
	case pcmk_action_start:
	case pcmk_action_notify:
	case pcmk_action_promote:
	case pcmk_action_demote:
	remapped_task = first_task + 1;
	break;
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_notified:
	case pcmk_action_promoted:
	case pcmk_action_demoted:
	remapped_task = first_task;
	break;
	case pcmk_action_monitor:
	case pcmk_action_shutdown:
	case pcmk_action_fence:
	break;
	default:
	crm_err("Unknown action '%s' in ordering", first_task_str);
	break;
	}

	if (remapped_task != pcmk_action_unspecified) {
	/* If a clone or bundle has notifications enabled, the ordering will be
	* relative to when notifications have been sent for the remapped task.
	*/
	if (pcmk_is_set(first_rsc->flags, pcmk_rsc_notify)
	&& (pe_rsc_is_clone(first_rsc) \|\| pe_rsc_is_bundled(first_rsc))) {
	uuid = pcmk__notify_key(rid, "confirmed-post",
	task2text(remapped_task));
	} else {
	uuid = pcmk__op_key(rid, task2text(remapped_task), 0);
	}
	pe_rsc_trace(first_rsc,
	"Remapped action UUID %s to %s for ordering purposes",
	first_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(first_uuid);
	CRM_ASSERT(uuid != NULL);
	}
	free(first_task_str);
	free(rid);
	return uuid;
	}

	/*!
	* \internal
	* \brief Get actual action that should be used with an ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the first action in an ordering, this returns the
	* the action that should actually be used for ordering (for example, the
	* started action instead of the start action).
	*
	* \param[in] action First action in an ordering
	*
	* \return Actual action that should be used for the ordering
	*/
	static pe_action_t *
	action_for_ordering(pe_action_t *action)
	{
	pe_action_t *result = action;
	pe_resource_t *rsc = action->rsc;

	if ((rsc != NULL) && (rsc->variant >= pcmk_rsc_variant_group)
	&& (action->uuid != NULL)) {
	char *uuid = action_uuid_for_ordering(action->uuid, rsc);

	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_warn("Not remapping %s to %s because %s does not have "
	"remapped action", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Wrapper for update_ordered_actions() method for readability
	*
	* \param[in,out] rsc Resource to call method for
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pcmk_action_optional to affect only
	* mandatory actions, and pe_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static inline uint32_t
	update(pe_resource_t rsc, pe_action_t first, pe_action_t *then,
	const pe_node_t *node, uint32_t flags, uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	return rsc->cmds->update_ordered_actions(first, then, node, flags, filter,
	type, data_set);
	}

	/*!
	* \internal
	* \brief Update flags for ordering's actions appropriately for ordering's flags
	*
	* \param[in,out] first First action in an ordering
	* \param[in,out] then Then action in an ordering
	* \param[in] first_flags Action flags for \p first for ordering purposes
	* \param[in] then_flags Action flags for \p then for ordering purposes
	* \param[in,out] order Action wrapper for \p first in ordering
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags
	*/
	static uint32_t
	update_action_for_ordering_flags(pe_action_t first, pe_action_t then,
	uint32_t first_flags, uint32_t then_flags,
	pe_action_wrapper_t *order,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;

	/* The node will only be used for clones. If interleaved, node will be NULL,
	* otherwise the ordering scope will be limited to the node. Normally, the
	* whole 'then' clone should restart if 'first' is restarted, so then->node
	* is needed.
	*/
	pe_node_t *node = then->node;

	if (pcmk_is_set(order->type, pe_order_implies_then_on_node)) {
	/* For unfencing, only instances of 'then' on the same node as 'first'
	* (the unfencing operation) should restart, so reset node to
	* first->node, at which point this case is handled like a normal
	- * pe_order_implies_then.
	+ * pcmk__ar_first_implies_then.
	*/
	pe__clear_order_flags(order->type, pe_order_implies_then_on_node);
	- pe__set_order_flags(order->type, pe_order_implies_then);
	+ pe__set_order_flags(order->type, pcmk__ar_first_implies_then);
	node = first->node;
	pe_rsc_trace(then->rsc,
	"%s then %s: mapped pe_order_implies_then_on_node to "
	- "pe_order_implies_then on %s",
	+ "pcmk__ar_first_implies_then on %s",
	first->uuid, then->uuid, pe__node_name(node));
	}

	- if (pcmk_is_set(order->type, pe_order_implies_then)) {
	+ if (pcmk_is_set(order->type, pcmk__ar_first_implies_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	- pcmk_action_optional, pe_order_implies_then,
	+ pcmk_action_optional, pcmk__ar_first_implies_then,
	data_set);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)) {
	pe__clear_action_flags(then, pcmk_action_optional);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	- pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_then",
	+ pe_rsc_trace(then->rsc,
	+ "%s then %s: %s after pcmk__ar_first_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_restart) && (then->rsc != NULL)) {
	enum pe_action_flags restart = pcmk_action_optional
	\|pcmk_action_runnable;

	changed \|= update(then->rsc, first, then, node, first_flags, restart,
	pe_order_restart, data_set);
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_restart",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_then_implies_first)) {
	if (first->rsc != NULL) {
	changed \|= update(first->rsc, first, then, node, first_flags,
	pcmk_action_optional, pcmk__ar_then_implies_first,
	data_set);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(first->flags, pcmk_action_runnable)) {
	pe__clear_action_flags(first, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	}
	pe_rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_then_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_promoted_implies_first)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	pcmk_action_optional,
	pe_order_promoted_implies_first, data_set);
	}
	pe_rsc_trace(then->rsc,
	"%s then %s: %s after pe_order_promoted_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_one_or_more)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pe_order_one_or_more,
	data_set);

	} else if (pcmk_is_set(first_flags, pcmk_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pe__set_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_one_or_more",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_probe) && (then->rsc != NULL)) {
	if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& (first->rsc->running_on != NULL)) {

	pe_rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	order->type = pcmk__ar_none;
	} else {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pe_order_runnable_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_runnable_left)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pe_order_runnable_left,
	data_set);

	} else if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pe__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_runnable_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_migratable)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional,
	pe_order_implies_first_migratable, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after "
	"pe_order_implies_first_migratable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_pseudo_left)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional, pe_order_pseudo_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_pseudo_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_ordered)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_ordered, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_ordered",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_asymmetrical)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pe_order_asymmetrical,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_asymmetrical",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk_is_set(order->type, pe_order_implies_then_printed)
	&& !pcmk_is_set(first_flags, pcmk_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pe__set_action_flags(then, pcmk_action_always_in_graph);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_printed)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pe__set_action_flags(first, pcmk_action_always_in_graph);
	// Don't bother marking 'first' as changed just for this
	}

	- if (pcmk_any_flags_set(order->type, pe_order_implies_then
	+ if (pcmk_any_flags_set(order->type, pcmk__ar_first_implies_then
	\|pcmk__ar_then_implies_first
	\|pe_order_restart)
	&& (first->rsc != NULL)
	&& !pcmk_is_set(first->rsc->flags, pcmk_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pcmk_rsc_blocked)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk__str_eq(first->task, PCMK_ACTION_STOP, pcmk__str_none)) {

	if (pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pe__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Update an action's flags for all orderings where it is "then"
	*
	* \param[in,out] then Action to update
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__update_action_for_orderings(pe_action_t then, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	uint32_t changed = pcmk__updated_none;
	int last_flags = then->flags;

	pe_rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	/* Initialize current known "runnable before" actions. As
	* update_action_for_ordering_flags() is called for each of then's
	* before actions, this number will increment as runnable 'first'
	* actions are encountered.
	*/
	then->runnable_before = 0;

	if (then->required_runnable_before == 0) {
	/* @COMPAT This ordering constraint uses the deprecated
	* "require-all=false" attribute. Treat it like "clone-min=1".
	*/
	then->required_runnable_before = 1;
	}

	/* The pe_order_one_or_more clause of update_action_for_ordering_flags()
	* (called below) will reset runnable if appropriate.
	*/
	pe__clear_action_flags(then, pcmk_action_runnable);
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;
	pe_action_t *first = other->action;

	pe_node_t *then_node = then->node;
	pe_node_t *first_node = first->node;

	if ((first->rsc != NULL)
	&& (first->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(first->task, PCMK_ACTION_START, pcmk__str_none)) {

	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node != NULL) {
	pe_rsc_trace(first->rsc, "Found %s for 'first' %s",
	pe__node_name(first_node), first->uuid);
	}
	}

	if ((then->rsc != NULL)
	&& (then->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)) {

	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node != NULL) {
	pe_rsc_trace(then->rsc, "Found %s for 'then' %s",
	pe__node_name(then_node), then->uuid);
	}
	}

	// Disable constraint if it only applies when on same node, but isn't
	if (pcmk_is_set(other->type, pe_order_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& !pe__same_node(first_node, then_node)) {

	pe_rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: "
	"not same node",
	other->action->uuid, pe__node_name(first_node),
	then->uuid, pe__node_name(then_node));
	other->type = pcmk__ar_none;
	continue;
	}

	pcmk__clear_updated_flags(changed, then, pcmk__updated_first);

	if ((first->rsc != NULL)
	&& pcmk_is_set(other->type, pe_order_then_cancels_first)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pe__set_action_flags(other->action, pcmk_action_optional);
	if (!strcmp(first->task, PCMK_ACTION_RELOAD_AGENT)) {
	pe__clear_resource_flags(first->rsc, pcmk_rsc_reload);
	}
	}

	if ((first->rsc != NULL) && (then->rsc != NULL)
	&& (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
	first = action_for_ordering(first);
	}
	if (first != other->action) {
	pe_rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	pe_rsc_trace(then->rsc,
	"%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
	first->uuid, first->flags, then->uuid, then->flags,
	other->type, action_node_str(first));

	if (first == other->action) {
	/* 'first' was not remapped (e.g. from 'start' to 'running'), which
	* could mean it is a non-resource action, a primitive resource
	* action, or already expanded.
	*/
	uint32_t first_flags, then_flags;

	first_flags = action_flags_for_ordering(first, then_node);
	then_flags = action_flags_for_ordering(then, first_node);

	changed \|= update_action_for_ordering_flags(first, then,
	first_flags, then_flags,
	other, data_set);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	other->type = pcmk__ar_none;
	}


	if (pcmk_is_set(changed, pcmk__updated_first)) {
	crm_trace("Re-processing %s and its 'after' actions "
	"because it changed", first->uuid);
	for (GList *lpc2 = first->actions_after; lpc2 != NULL;
	lpc2 = lpc2->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc2->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	pcmk__update_action_for_orderings(first, data_set);
	}
	}

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	if (last_flags == then->flags) {
	pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
	} else {
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	}
	}

	if (pcmk_is_set(changed, pcmk__updated_then)) {
	crm_trace("Re-processing %s and its 'after' actions because it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pcmk__block_colocation_dependents(then);
	}
	pcmk__update_action_for_orderings(then, data_set);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	}
	}

	static inline bool
	is_primitive_action(const pe_action_t *action)
	{
	return (action != NULL) && (action->rsc != NULL)
	&& (action->rsc->variant == pcmk_rsc_variant_primitive);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in,out] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pe__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, false); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Update actions in an asymmetric ordering
	*
	* If the "first" action in an asymmetric ordering is unrunnable, make the
	* "second" action unrunnable as well, if appropriate.
	*
	* \param[in] first 'First' action in an asymmetric ordering
	* \param[in,out] then 'Then' action in an asymmetric ordering
	*/
	static void
	handle_asymmetric_ordering(const pe_action_t first, pe_action_t then)
	{
	/* Only resource actions after an unrunnable 'first' action need updates for
	* asymmetric ordering.
	*/
	if ((then->rsc == NULL)
	\|\| pcmk_is_set(first->flags, pcmk_action_runnable)) {
	return;
	}

	// Certain optional 'then' actions are unaffected by unrunnable 'first'
	if (pcmk_is_set(then->flags, pcmk_action_optional)) {
	enum rsc_role_e then_rsc_role = then->rsc->fns->state(then->rsc, TRUE);

	if ((then_rsc_role == pcmk_role_stopped)
	&& pcmk__str_eq(then->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* If 'then' should stop after 'first' but is already stopped, the
	* ordering is irrelevant.
	*/
	return;
	} else if ((then_rsc_role >= pcmk_role_started)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)
	&& pe__rsc_running_on_only(then->rsc, then->node)) {
	/* Similarly if 'then' should start after 'first' but is already
	* started on a single node.
	*/
	return;
	}
	}

	// 'First' can't run, so 'then' can't either
	clear_action_flag_because(then, pcmk_action_optional, first);
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in,out] first 'First' action in an ordering with pe_restart_order
	* \param[in,out] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What action flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then, uint32_t filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pcmk_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (ignored)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;
	uint32_t then_flags = 0U;
	uint32_t first_flags = 0U;

	CRM_ASSERT((first != NULL) && (then != NULL) && (data_set != NULL));

	then_flags = then->flags;
	first_flags = first->flags;
	if (pcmk_is_set(type, pe_order_asymmetrical)) {
	handle_asymmetric_ordering(first, then);
	}

	if (pcmk_is_set(type, pcmk__ar_then_implies_first)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& pcmk_is_set(first_flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	if (pcmk_is_set(flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pe_order_promoted_implies_first)
	&& (then->rsc != NULL) && (then->rsc->role == pcmk_role_promoted)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	clear_action_flag_because(first, pcmk_action_optional, then);

	if (pcmk_is_set(first->flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pcmk_action_optional)) {

	if (!pcmk_all_flags_set(then->flags, pcmk_action_migratable
	\|pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_runnable, then);
	}

	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}
	}

	if (pcmk_is_set(type, pe_order_pseudo_left)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_migratable, first);
	pe__clear_action_flags(then, pcmk_action_pseudo);
	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)
	&& !pcmk_is_set(flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_runnable, first);
	clear_action_flag_because(then, pcmk_action_migratable, first);
	}

	- if (pcmk_is_set(type, pe_order_implies_then)
	+ if (pcmk_is_set(type, pcmk__ar_first_implies_then)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_migratable)) {

	clear_action_flag_because(then, pcmk_action_optional, first);
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid, pe__node_name(then->node),
	then->flags, then_flags, first->uuid, first->flags);

	if ((then->rsc != NULL) && (then->rsc->parent != NULL)) {
	// Required to handle "X_stop then X_start" for cloned groups
	pcmk__update_action_for_orderings(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	pe_rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid, pe__node_name(first->node),
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Trace-log an action (optionally with its dependent actions)
	*
	* \param[in] pre_text If not NULL, prefix the log with this plus ": "
	* \param[in] action Action to log
	* \param[in] details If true, recursively log dependent actions
	*/
	void
	pcmk__log_action(const char pre_text, const pe_action_t action, bool details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	CRM_CHECK(action != NULL, return);

	if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	}
	}

	switch (text2task(action->task)) {
	case pcmk_action_fence:
	case pcmk_action_shutdown:
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	const GList *iter = NULL;
	const pe_action_wrapper_t *other = NULL;

	crm_trace("\t\t====== Preceding Actions");
	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	other = (const pe_action_wrapper_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== Subsequent Actions");
	for (iter = action->actions_after; iter != NULL; iter = iter->next) {
	other = (const pe_action_wrapper_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before),
	g_list_length(action->actions_after));
	}
	}

	/*!
	* \internal
	* \brief Create a new shutdown action for a node
	*
	* \param[in,out] node Node being shut down
	*
	* \return Newly created shutdown action for \p node
	*/
	pe_action_t *
	pcmk__new_shutdown_action(pe_node_t *node)
	{
	char *shutdown_id = NULL;
	pe_action_t *shutdown_op = NULL;

	CRM_ASSERT(node != NULL);

	shutdown_id = crm_strdup_printf("%s-%s", PCMK_ACTION_DO_SHUTDOWN,
	node->details->uname);

	shutdown_op = custom_action(NULL, shutdown_id, PCMK_ACTION_DO_SHUTDOWN,
	node, FALSE, TRUE, node->details->data_set);

	pcmk__order_stops_before_shutdown(node, shutdown_op);
	add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	return shutdown_op;
	}

	/*!
	* \internal
	* \brief Calculate and add an operation digest to XML
	*
	* Calculate an operation digest, which enables us to later determine when a
	* restart is needed due to the resource's parameters being changed, and add it
	* to given XML.
	*
	* \param[in] op Operation result from executor
	* \param[in,out] update XML to add digest to
	*/
	static void
	add_op_digest_to_xml(const lrmd_event_data_t op, xmlNode update)
	{
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}
	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, NULL);
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	crm_trace("Creating history XML for %s-interval %s action for %s on %s "
	"(DC version: %s, origin: %s)",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	((node == NULL)? "no node" : node), caller_version, origin);

	task = op->op_type;

	/* Record a successful agent reload as a start, and a failed one as a
	* monitor, to make life easier for the scheduler when determining the
	* current state.
	*
	* @COMPAT We should check "reload" here only if the operation was for a
	* pre-OCF-1.1 resource agent, but we don't know that here, and we should
	* only ever get results for actions scheduled by us, so we can reasonably
	* assume any "reload" is actually a pre-1.1 agent reload.
	*/
	if (pcmk__str_any_of(task, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
	NULL)) {
	if (op->op_status == PCMK_EXEC_DONE) {
	task = PCMK_ACTION_START;
	} else {
	task = PCMK_ACTION_MONITOR;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_EXEC_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	}

	/* Migration history is preserved separately, which usually matters for
	* multiple nodes and is important for future cluster transitions.
	*/
	} else if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	op_id = strdup(key);

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	// Ensure 'last' gets updated, in case record-pending is true
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if (magic == NULL) {
	magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
	(const char *) op->user_data);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, pcmk__s(exit_reason, ""));
	crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); // For context during triage

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	add_op_digest_to_xml(op, xml_op);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	/*!
	* \internal
	* \brief Check whether an action shutdown-locks a resource to a node
	*
	* If the shutdown-lock cluster property is set, resources will not be recovered
	* on a different node if cleanly stopped, and may start only on that same node.
	* This function checks whether that applies to a given action, so that the
	* transition graph can be marked appropriately.
	*
	* \param[in] action Action to check
	*
	* \return true if \p action locks its resource to the action's node,
	* otherwise false
	*/
	bool
	pcmk__action_locks_rsc_to_node(const pe_action_t *action)
	{
	// Only resource actions taking place on resource's lock node are locked
	if ((action == NULL) \|\| (action->rsc == NULL)
	\|\| !pe__same_node(action->node, action->rsc->lock_node)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && (action->task != NULL)
	&& (strcmp(action->task, PCMK_ACTION_STOP) != 0)) {
	return false;
	}

	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pe_action_wrapper_t action_wrapper2 = (const pe_action_wrapper_t )a;
	const pe_action_wrapper_t action_wrapper1 = (const pe_action_wrapper_t )b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}
	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Remove any duplicate action inputs, merging action flags
	*
	* \param[in,out] action Action whose inputs should be checked
	*/
	void
	pcmk__deduplicate_action_inputs(pe_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pe_action_wrapper_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) item->data;

	next = item->next;
	if ((last_input != NULL)
	&& (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Output all scheduled actions
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__output_actions(pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;

	// Output node (non-resource) actions
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) iter->data;

	if (action->rsc != NULL) {
	continue; // Resource actions will be output later

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue; // This action was not scheduled
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	task = strdup("Shutdown");

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta,
	"stonith_action");

	task = crm_strdup_printf("Fence (%s)", op);

	} else {
	continue; // Don't display other node action types
	}

	if (pe__is_guest_node(action->node)) {
	const pe_resource_t *remote = action->node->details->remote_rsc;

	node_name = crm_strdup_printf("%s (resource: %s)",
	pe__node_name(action->node),
	remote->container->id);
	} else if (action->node != NULL) {
	node_name = crm_strdup_printf("%s", pe__node_name(action->node));
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}

	// Output resource actions
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->output_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether action from resource history is still in configuration
	*
	* \param[in] rsc Resource that action is for
	* \param[in] task Action's name
	* \param[in] interval_ms Action's interval (in milliseconds)
	*
	* \return true if action is still in resource configuration, otherwise false
	*/
	static bool
	action_in_config(const pe_resource_t rsc, const char task, guint interval_ms)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	bool config = (find_rsc_op_entry(rsc, key) != NULL);

	free(key);
	return config;
	}

	/*!
	* \internal
	* \brief Get action name needed to compare digest for configuration changes
	*
	* \param[in] task Action name from history
	* \param[in] interval_ms Action interval (in milliseconds)
	*
	* \return Action name whose digest should be compared
	*/
	static const char *
	task_for_digest(const char *task, guint interval_ms)
	{
	/* Certain actions need to be compared against the parameters used to start
	* the resource.
	*/
	if ((interval_ms == 0)
	&& pcmk__str_any_of(task, PCMK_ACTION_MONITOR, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_PROMOTE, NULL)) {
	task = PCMK_ACTION_START;
	}
	return task;
	}

	/*!
	* \internal
	* \brief Check whether only sanitized parameters to an action changed
	*
	* When collecting CIB files for troubleshooting, crm_report will mask
	* sensitive resource parameters. If simulations were run using that, affected
	* resources would appear to need a restart, which would complicate
	* troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
	* parameters. This function used that digest to check whether only masked
	* parameters are different.
	*
	* \param[in] xml_op Resource history entry with secure digest
	* \param[in] digest_data Operation digest information being compared
	* \param[in] data_set Cluster working set
	*
	* \return true if only sanitized parameters changed, otherwise false
	*/
	static bool
	only_sanitized_changed(const xmlNode *xml_op,
	const op_digest_cache_t *digest_data,
	const pe_working_set_t *data_set)
	{
	const char *digest_secure = NULL;

	if (!pcmk_is_set(data_set->flags, pcmk_sched_sanitized)) {
	// The scheduler is not being run as a simulation
	return false;
	}

	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);

	return (digest_data->rc != RSC_DIGEST_MATCH) && (digest_secure != NULL)
	&& (digest_data->digest_secure_calc != NULL)
	&& (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
	}

	/*!
	* \internal
	* \brief Force a restart due to a configuration change
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action whose configuration changed
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where resource should be restarted
	*/
	static void
	force_restart(pe_resource_t rsc, const char task, guint interval_ms,
	pe_node_t *node)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	pe_action_t *required = custom_action(rsc, key, task, NULL, FALSE, TRUE,
	rsc->cluster);

	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, node, "Device parameters changed", NULL,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Schedule a reload of a resource on a node
	*
	* \param[in,out] data Resource to reload
	* \param[in] user_data Where resource should be reloaded
	*/
	static void
	schedule_reload(gpointer data, gpointer user_data)
	{
	pe_resource_t *rsc = data;
	const pe_node_t *node = user_data;
	pe_action_t *reload = NULL;

	// For collective resources, just call recursively for children
	if (rsc->variant > pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, schedule_reload, user_data);
	return;
	}

	// Skip the reload in certain situations
	if ((node == NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pe_rsc_trace(rsc, "Skip reload of %s:%s%s %s",
	rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " unmanaged",
	pcmk_is_set(rsc->flags, pcmk_rsc_failed)? " failed" : "",
	(node == NULL)? "inactive" : node->details->uname);
	return;
	}

	/* If a resource's configuration changed while a start was pending,
	* force a full restart instead of a reload.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	pe_rsc_trace(rsc, "%s: preventing agent reload because start pending",
	rsc->id);
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE, TRUE,
	rsc->cluster);
	return;
	}

	// Schedule the reload
	pe__set_resource_flags(rsc, pcmk_rsc_reload);
	reload = custom_action(rsc, reload_key(rsc), PCMK_ACTION_RELOAD_AGENT, node,
	FALSE, TRUE, rsc->cluster);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	// Set orderings so that a required stop or demote cancels the reload
	pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pcmk__ar_ordered\|pe_order_then_cancels_first,
	rsc->cluster);
	pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pcmk__ar_ordered\|pe_order_then_cancels_first,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Handle any configuration change for an action
	*
	* Given an action from resource history, if the resource's configuration
	* changed since the action was done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, etc.).
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in,out] node Node that action was on
	* \param[in] xml_op Action XML from resource history
	*
	* \return true if action configuration changed, otherwise false
	*/
	bool
	pcmk__check_action_config(pe_resource_t rsc, pe_node_t node,
	const xmlNode *xml_op)
	{
	guint interval_ms = 0;
	const char *task = NULL;
	const op_digest_cache_t *digest_data = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
	return false);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	CRM_CHECK(task != NULL, return false);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	// If this is a recurring action, check whether it has been orphaned
	if (interval_ms > 0) {
	if (action_in_config(rsc, task, interval_ms)) {
	pe_rsc_trace(rsc, "%s-interval %s for %s on %s is in configuration",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_cancel_removed_actions)) {
	pcmk__schedule_cancel(rsc,
	crm_element_value(xml_op,
	XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "orphan");
	return true;
	} else {
	pe_rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	return true;
	}
	}

	crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	task = task_for_digest(task, interval_ms);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, rsc->cluster);

	if (only_sanitized_changed(xml_op, digest_data, rsc->cluster)) {
	if (!pcmk__is_daemon && (rsc->cluster->priv != NULL)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->info(out,
	"Only 'private' parameters to %s-interval %s for %s "
	"on %s changed: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node),
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}
	return false;
	}

	switch (digest_data->rc) {
	case RSC_DIGEST_RESTART:
	crm_log_xml_debug(digest_data->params_restart, "params:restart");
	force_restart(rsc, task, interval_ms, node);
	return true;

	case RSC_DIGEST_ALL:
	case RSC_DIGEST_UNKNOWN:
	// Changes that can potentially be handled by an agent reload

	if (interval_ms > 0) {
	/* Recurring actions aren't reloaded per se, they are just
	* re-scheduled so the next run uses the new parameters.
	* The old instance will be cancelled automatically.
	*/
	crm_log_xml_debug(digest_data->params_all, "params:reschedule");
	pcmk__reschedule_recurring(rsc, task, interval_ms, node);

	} else if (crm_element_value(xml_op,
	XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
	// Agent supports reload, so use it
	trigger_unfencing(rsc, node,
	"Device parameters changed (reload)", NULL,
	rsc->cluster);
	crm_log_xml_debug(digest_data->params_all, "params:reload");
	schedule_reload((gpointer) rsc, (gpointer) node);

	} else {
	pe_rsc_trace(rsc,
	"Restarting %s "
	"because agent doesn't support reload", rsc->id);
	crm_log_xml_debug(digest_data->params_restart,
	"params:restart");
	force_restart(rsc, task, interval_ms, node);
	}
	return true;

	default:
	break;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create a list of resource's action history entries, sorted by call ID
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[out] start_index Where to store index of start-like action, if any
	* \param[out] stop_index Where to store index of stop action, if any
	*/
	static GList *
	rsc_history_as_list(const xmlNode rsc_entry, int start_index, int *stop_index)
	{
	GList *ops = NULL;

	for (xmlNode *rsc_op = first_named_child(rsc_entry, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
	ops = g_list_prepend(ops, rsc_op);
	}
	ops = g_list_sort(ops, sort_op_by_callid);
	calculate_active_ops(ops, start_index, stop_index);
	return ops;
	}

	/*!
	* \internal
	* \brief Process a resource's action history from the CIB status
	*
	* Given a resource's action history, if the resource's configuration
	* changed since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[in,out] rsc Resource whose history is being processed
	* \param[in,out] node Node whose history is being processed
	*/
	static void
	process_rsc_history(const xmlNode rsc_entry, pe_resource_t rsc,
	pe_node_t *node)
	{
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;
	GList *sorted_op_list = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pe_rsc_is_anon_clone(pe__const_top_resource(rsc, false))) {
	pe_rsc_trace(rsc,
	"Skipping configuration check "
	"for orphaned clone instance %s",
	rsc->id);
	} else {
	pe_rsc_trace(rsc,
	"Skipping configuration check and scheduling clean-up "
	"for orphaned resource %s", rsc->id);
	pcmk__schedule_cleanup(rsc, node, false);
	}
	return;
	}

	if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}
	pe_rsc_trace(rsc,
	"Skipping configuration check for %s "
	"because no longer active on %s",
	rsc->id, pe__node_name(node));
	return;
	}

	pe_rsc_trace(rsc, "Checking for configuration changes for %s on %s",
	rsc->id, pe__node_name(node));

	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}

	sorted_op_list = rsc_history_as_list(rsc_entry, &start_index, &stop_index);
	if (start_index < stop_index) {
	return; // Resource is stopped
	}

	for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	xmlNode rsc_op = (xmlNode ) iter->data;
	const char *task = NULL;
	guint interval_ms = 0;

	if (++offset < start_index) {
	// Skip actions that happened before a start
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0)
	&& (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)
	\|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	pcmk__schedule_cancel(rsc,
	crm_element_value(rsc_op,
	XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "maintenance mode");

	} else if ((interval_ms > 0)
	\|\| pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't assigned resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pcmk__check_active,
	rsc->cluster);

	} else if (pcmk__check_action_config(rsc, node, rsc_op)
	&& (pe_get_failcount(node, rsc, NULL, pe_fc_effective,
	NULL) != 0)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	rsc->cluster);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	/*!
	* \internal
	* \brief Process a node's action history from the CIB status
	*
	* Given a node's resource history, if the resource's configuration changed
	* since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] node Node whose history is being processed
	* \param[in] lrm_rscs Node's <lrm_resources> from CIB status XML
	*/
	static void
	process_node_history(pe_node_t node, const xmlNode lrm_rscs)
	{
	crm_trace("Processing node history for %s", pe__node_name(node));
	for (const xmlNode *rsc_entry = first_named_child(lrm_rscs,
	XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	if (rsc_entry->children != NULL) {
	GList *result = pcmk__rscs_matching_id(ID(rsc_entry),
	node->details->data_set);

	for (GList *iter = result; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	process_rsc_history(rsc_entry, rsc, node);
	}
	}
	g_list_free(result);
	}
	}
	}

	// XPath to find a node's resource history
	#define XPATH_NODE_HISTORY "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES

	/*!
	* \internal
	* \brief Process any resource configuration changes in the CIB status
	*
	* Go through all nodes' resource history, and if a resource's configuration
	* changed since its actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__handle_rsc_config_changes(pe_working_set_t *data_set)
	{
	crm_trace("Check resource and action configuration for changes");

	/* Rather than iterate through the status section, iterate through the nodes
	* and search for the appropriate status subsection for each. This skips
	* orphaned nodes and lets us eliminate some cases before searching the XML.
	*/
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	/* Don't bother checking actions for a node that can't run actions ...
	* unless it's in maintenance mode, in which case we still need to
	* cancel any existing recurring monitors.
	*/
	if (node->details->maintenance
	\|\| pcmk__node_available(node, false, false)) {

	char *xpath = NULL;
	xmlNode *history = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->details->uname);
	history = get_xpath_object(xpath, data_set->input, LOG_NEVER);
	free(xpath);

	process_node_history(node, history);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_fencing.c b/lib/pacemaker/pcmk_sched_fencing.c
	index e626342f53..58e785c577 100644
	--- a/lib/pacemaker/pcmk_sched_fencing.c
	+++ b/lib/pacemaker/pcmk_sched_fencing.c
	@@ -1,496 +1,496 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a resource is known on a particular node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return TRUE if resource (or parent if an anonymous clone) is known
	*/
	static bool
	rsc_is_known_on(const pe_resource_t rsc, const pe_node_t node)
	{
	if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	return TRUE;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& pe_rsc_is_anon_clone(rsc->parent)
	&& (g_hash_table_lookup(rsc->parent->known_on,
	node->details->id) != NULL)) {
	/* We check only the parent, not the uber-parent, because we cannot
	* assume that the resource is known if it is in an anonymously cloned
	* group (which may be only partially known).
	*/
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Order a resource's start and promote actions relative to fencing
	*
	* \param[in,out] rsc Resource to be ordered
	* \param[in,out] stonith_op Fence action
	*/
	static void
	order_start_vs_fencing(pe_resource_t rsc, pe_action_t stonith_op)
	{
	pe_node_t *target;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pe_action_t *action = iter->data;

	switch (action->needs) {
	case pcmk_requires_nothing:
	// Anything other than start or promote requires nothing
	break;

	case pcmk_requires_fencing:
	order_actions(stonith_op, action, pcmk__ar_ordered);
	break;

	case pcmk_requires_quorum:
	if (pcmk__str_eq(action->task, PCMK_ACTION_START,
	pcmk__str_none)
	&& (g_hash_table_lookup(rsc->allowed_nodes,
	target->details->id) != NULL)
	&& !rsc_is_known_on(rsc, target)) {

	/* If we don't know the status of the resource on the node
	* we're about to shoot, we have to assume it may be active
	* there. Order the resource start after the fencing. This
	* is analogous to waiting for all the probes for a resource
	* to complete before starting it.
	*
	* The most likely explanation is that the DC died and took
	* its status with it.
	*/
	pe_rsc_debug(rsc, "Ordering %s after %s recovery",
	action->uuid, pe__node_name(target));
	order_actions(stonith_op, action,
	pcmk__ar_ordered\|pe_order_runnable_left);
	}
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Order a resource's stop and demote actions relative to fencing
	*
	* \param[in,out] rsc Resource to be ordered
	* \param[in,out] stonith_op Fence action
	*/
	static void
	order_stop_vs_fencing(pe_resource_t rsc, pe_action_t stonith_op)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool order_implicit = false;

	pe_resource_t *top = uber_parent(rsc);
	pe_action_t *parent_stop = NULL;
	pe_node_t *target;

	CRM_CHECK(stonith_op && stonith_op->node, return);
	target = stonith_op->node;

	/* Get a list of stop actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, PCMK_ACTION_STOP, FALSE);

	/* If resource requires fencing, implicit actions must occur after fencing.
	*
	* Implied stops and demotes of resources running on guest nodes are always
	* ordered after fencing, even if the resource does not require fencing,
	* because guest node "fencing" is actually just a resource stop.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	\|\| pe__is_guest_node(target)) {

	order_implicit = true;
	}

	if (action_list && order_implicit) {
	parent_stop = find_first_action(top->actions, NULL, PCMK_ACTION_STOP,
	NULL);
	}

	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t *action = iter->data;

	// The stop would never complete, so convert it into a pseudo-action.
	pe__set_action_flags(action, pcmk_action_pseudo\|pcmk_action_runnable);

	if (order_implicit) {
	/* Order the stonith before the parent stop (if any).
	*
	* Also order the stonith before the resource stop, unless the
	* resource is inside a bundle -- that would cause a graph loop.
	* We can rely on the parent stop's ordering instead.
	*
	* User constraints must not order a resource in a guest node
	* relative to the guest node container resource. The
	* pe_order_preserve flag marks constraints as generated by the
	* cluster and thus immune to that check (and is irrelevant if
	* target is not a guest).
	*/
	if (!pe_rsc_is_bundled(rsc)) {
	order_actions(stonith_op, action, pe_order_preserve);
	}
	order_actions(stonith_op, parent_stop, pe_order_preserve);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_notice("Stop of failed resource %s is implicit %s %s is fenced",
	rsc->id, (order_implicit? "after" : "because"),
	pe__node_name(target));
	} else {
	crm_info("%s is implicit %s %s is fenced",
	action->uuid, (order_implicit? "after" : "because"),
	pe__node_name(target));
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_notify)) {
	pe__order_notifs_after_fencing(action, rsc, stonith_op);
	}

	#if 0
	/* It might be a good idea to stop healthy resources on a node about to
	* be fenced, when possible.
	*
	* However, fencing must be done before a failed resource's
	* (pseudo-)stop action, so that could create a loop. For example, given
	* a group of A and B running on node N with a failed stop of B:
	*
	* fence N -> stop B (pseudo-op) -> stop A -> fence N
	*
	* The block below creates the stop A -> fence N ordering and therefore
	* must (at least for now) be disabled. Instead, run the block above and
	* treat all resources on N as B would be (i.e., as a pseudo-op after
	* the fencing).
	*
	* @TODO Maybe break the "A requires B" dependency in
	* pcmk__update_action_for_orderings() and use this block for healthy
	* resources instead of the above.
	*/
	crm_info("Moving healthy resource %s off %s before fencing",
	rsc->id, pe__node_name(node));
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL,
	strdup(PCMK_ACTION_STONITH), stonith_op,
	pcmk__ar_ordered, rsc->cluster);
	#endif
	}

	g_list_free(action_list);

	/* Get a list of demote actions potentially implied by the fencing */
	action_list = pe__resource_actions(rsc, target, PCMK_ACTION_DEMOTE, FALSE);

	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t *action = iter->data;

	if (!(action->node->details->online) \|\| action->node->details->unclean
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pe_rsc_info(rsc,
	"Demote of failed resource %s is implicit "
	"after %s is fenced",
	rsc->id, pe__node_name(target));
	} else {
	pe_rsc_info(rsc, "%s is implicit after %s is fenced",
	action->uuid, pe__node_name(target));
	}

	/* The demote would never complete and is now implied by the
	* fencing, so convert it into a pseudo-action.
	*/
	pe__set_action_flags(action,
	pcmk_action_pseudo\|pcmk_action_runnable);

	if (pe_rsc_is_bundled(rsc)) {
	// Recovery will be ordered as usual after parent's implied stop

	} else if (order_implicit) {
	order_actions(stonith_op, action,
	pe_order_preserve\|pcmk__ar_ordered);
	}
	}
	}

	g_list_free(action_list);
	}

	/*!
	* \internal
	* \brief Order resource actions properly relative to fencing
	*
	* \param[in,out] rsc Resource whose actions should be ordered
	* \param[in,out] stonith_op Fencing operation to be ordered against
	*/
	static void
	rsc_stonith_ordering(pe_resource_t rsc, pe_action_t stonith_op)
	{
	if (rsc->children) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t *child_rsc = iter->data;

	rsc_stonith_ordering(child_rsc, stonith_op);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping fencing constraints for unmanaged resource: %s",
	rsc->id);

	} else {
	order_start_vs_fencing(rsc, stonith_op);
	order_stop_vs_fencing(rsc, stonith_op);
	}
	}

	/*!
	* \internal
	* \brief Order all actions appropriately relative to a fencing operation
	*
	* Ensure start operations of affected resources are ordered after fencing,
	* imply stop and demote operations of affected resources by marking them as
	* pseudo-actions, etc.
	*
	* \param[in,out] stonith_op Fencing operation
	* \param[in,out] data_set Working set of cluster
	*/
	void
	pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set)
	{
	CRM_CHECK(stonith_op && data_set, return);
	for (GList *r = data_set->resources; r != NULL; r = r->next) {
	rsc_stonith_ordering((pe_resource_t *) r->data, stonith_op);
	}
	}

	/*!
	* \internal
	* \brief Order an action after unfencing
	*
	* \param[in] rsc Resource that action is for
	* \param[in,out] node Node that action is on
	* \param[in,out] action Action to be ordered after unfencing
	* \param[in] order Ordering flags
	*/
	void
	pcmk__order_vs_unfence(const pe_resource_t rsc, pe_node_t node,
	pe_action_t *action,
	enum pcmk__action_relation_flags order)
	{
	/* When unfencing is in use, we order unfence actions before any probe or
	* start of resources that require unfencing, and also of fence devices.
	*
	* This might seem to violate the principle that fence devices require
	* only quorum. However, fence agents that unfence often don't have enough
	* information to even probe or start unless the node is first unfenced.
	*/
	if ((pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_enable_unfencing))
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing)) {

	/* Start with an optional ordering. Requiring unfencing would result in
	* the node being unfenced, and all its resources being stopped,
	* whenever a new resource is added -- which would be highly suboptimal.
	*/
	pe_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, TRUE, NULL,
	FALSE, node->details->data_set);

	order_actions(unfence, action, order);

	if (!pcmk__node_unfenced(node)) {
	// But unfencing is required if it has never been done
	char *reason = crm_strdup_printf("required by %s %s",
	rsc->id, action->task);

	trigger_unfencing(NULL, node, reason, NULL,
	node->details->data_set);
	free(reason);
	}
	}
	}

	/*!
	* \internal
	* \brief Create pseudo-op for guest node fence, and order relative to it
	*
	* \param[in,out] node Guest node to fence
	*/
	void
	pcmk__fence_guest(pe_node_t *node)
	{
	pe_resource_t *container = NULL;
	pe_action_t *stop = NULL;
	pe_action_t *stonith_op = NULL;

	/* The fence action is just a label; we don't do anything differently for
	* off vs. reboot. We specify it explicitly, rather than let it default to
	* cluster's default action, because we are not _initiating_ fencing -- we
	* are creating a pseudo-event to describe fencing that is already occurring
	* by other means (container recovery).
	*/
	const char *fence_action = PCMK_ACTION_OFF;

	CRM_ASSERT(node != NULL);

	/* Check whether guest's container resource has any explicit stop or
	* start (the stop may be implied by fencing of the guest's host).
	*/
	container = node->details->remote_rsc->container;
	if (container) {
	stop = find_first_action(container->actions, NULL, PCMK_ACTION_STOP,
	NULL);

	if (find_first_action(container->actions, NULL, PCMK_ACTION_START,
	NULL)) {
	fence_action = PCMK_ACTION_REBOOT;
	}
	}

	/* Create a fence pseudo-event, so we have an event to order actions
	* against, and the controller can always detect it.
	*/
	stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean",
	FALSE, node->details->data_set);
	pe__set_action_flags(stonith_op, pcmk_action_pseudo\|pcmk_action_runnable);

	/* We want to imply stops/demotes after the guest is stopped, not wait until
	* it is restarted, so we always order pseudo-fencing after stop, not start
	* (even though start might be closer to what is done for a real reboot).
	*/
	if ((stop != NULL) && pcmk_is_set(stop->flags, pcmk_action_pseudo)) {
	pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE,
	NULL, FALSE,
	node->details->data_set);

	crm_info("Implying guest %s is down (action %d) after %s fencing",
	pe__node_name(node), stonith_op->id,
	pe__node_name(stop->node));
	order_actions(parent_stonith_op, stonith_op,
	- pe_order_runnable_left\|pe_order_implies_then);
	+ pe_order_runnable_left\|pcmk__ar_first_implies_then);

	} else if (stop) {
	order_actions(stop, stonith_op,
	- pe_order_runnable_left\|pe_order_implies_then);
	+ pe_order_runnable_left\|pcmk__ar_first_implies_then);
	crm_info("Implying guest %s is down (action %d) "
	"after container %s is stopped (action %d)",
	pe__node_name(node), stonith_op->id,
	container->id, stop->id);
	} else {
	/* If we're fencing the guest node but there's no stop for the guest
	* resource, we must think the guest is already stopped. However, we may
	* think so because its resource history was just cleaned. To avoid
	* unnecessarily considering the guest node down if it's really up,
	* order the pseudo-fencing after any stop of the connection resource,
	* which will be ordered after any container (re-)probe.
	*/
	stop = find_first_action(node->details->remote_rsc->actions, NULL,
	PCMK_ACTION_STOP, NULL);

	if (stop) {
	order_actions(stop, stonith_op, pcmk__ar_ordered);
	crm_info("Implying guest %s is down (action %d) "
	"after connection is stopped (action %d)",
	pe__node_name(node), stonith_op->id, stop->id);
	} else {
	/* Not sure why we're fencing, but everything must already be
	* cleanly stopped.
	*/
	crm_info("Implying guest %s is down (action %d) ",
	pe__node_name(node), stonith_op->id);
	}
	}

	// Order/imply other actions relative to pseudo-fence as with real fence
	pcmk__order_vs_fence(stonith_op, node->details->data_set);
	}

	/*!
	* \internal
	* \brief Check whether node has already been unfenced
	*
	* \param[in] node Node to check
	*
	* \return true if node has a nonzero #node-unfenced attribute (or none),
	* otherwise false
	*/
	bool
	pcmk__node_unfenced(const pe_node_t *node)
	{
	const char *unfenced = pe_node_attribute_raw(node, CRM_ATTR_UNFENCED);

	return !pcmk__str_eq(unfenced, "0", pcmk__str_null_matches);
	}

	/*!
	* \internal
	* \brief Order a resource's start and stop relative to unfencing of a node
	*
	* \param[in,out] data Node that could be unfenced
	* \param[in,out] user_data Resource to order
	*/
	void
	pcmk__order_restart_vs_unfence(gpointer data, gpointer user_data)
	{
	pe_node_t node = (pe_node_t ) data;
	pe_resource_t rsc = (pe_resource_t ) user_data;

	pe_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, true, NULL, false,
	rsc->cluster);

	crm_debug("Ordering any stops of %s before %s, and any starts after",
	rsc->id, unfence->uuid);

	/*
	* It would be more efficient to order clone resources once,
	* rather than order each instance, but ordering the instance
	* allows us to avoid unnecessary dependencies that might conflict
	* with user constraints.
	*
	* @TODO: This constraint can still produce a transition loop if the
	* resource has a stop scheduled on the node being unfenced, and
	* there is a user ordering constraint to start some other resource
	* (which will be ordered after the unfence) before stopping this
	* resource. An example is "start some slow-starting cloned service
	* before stopping an associated virtual IP that may be moving to
	* it":
	* stop this -> unfencing -> start that -> stop this
	*/
	pcmk__new_ordering(rsc, stop_key(rsc), NULL,
	NULL, strdup(unfence->uuid), unfence,
	pcmk__ar_ordered\|pe_order_same_node,
	rsc->cluster);

	pcmk__new_ordering(NULL, strdup(unfence->uuid), unfence,
	rsc, start_key(rsc), NULL,
	pe_order_implies_then_on_node\|pe_order_same_node,
	rsc->cluster);
	}
	diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c
	index b0f1e48385..94ea7b28b3 100644
	--- a/lib/pacemaker/pcmk_sched_group.c
	+++ b/lib/pacemaker/pcmk_sched_group.c
	@@ -1,949 +1,950 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>

	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Assign a group resource to a node
	*
	* \param[in,out] rsc Group resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a child of \p rsc can't be
	* assigned to a node, set the child's next role to
	* stopped and update existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pe_node_t *
	pcmk__group_assign(pe_resource_t rsc, const pe_node_t prefer,
	bool stop_if_fail)
	{
	pe_node_t *first_assigned_node = NULL;
	pe_resource_t *first_member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return rsc->allocated_to; // Assignment already done
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pe_rsc_debug(rsc, "Assignment dependency loop detected involving %s",
	rsc->id);
	return NULL;
	}

	if (rsc->children == NULL) {
	// No members to assign
	pe__clear_resource_flags(rsc, pcmk_rsc_unassigned);
	return NULL;
	}

	pe__set_resource_flags(rsc, pcmk_rsc_assigning);
	first_member = (pe_resource_t *) rsc->children->data;
	rsc->role = first_member->role;

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;
	pe_node_t *node = NULL;

	pe_rsc_trace(rsc, "Assigning group %s member %s",
	rsc->id, member->id);
	node = member->cmds->assign(member, prefer, stop_if_fail);
	if (first_assigned_node == NULL) {
	first_assigned_node = node;
	}
	}

	pe__set_next_role(rsc, first_member->next_role, "first group member");
	pe__clear_resource_flags(rsc, pcmk_rsc_assigning\|pcmk_rsc_unassigned);

	if (!pe__group_flag_is_set(rsc, pe__group_colocated)) {
	return NULL;
	}
	return first_assigned_node;
	}

	/*!
	* \internal
	* \brief Create a pseudo-operation for a group as an ordering point
	*
	* \param[in,out] group Group resource to create action for
	* \param[in] action Action name
	*
	* \return Newly created pseudo-operation
	*/
	static pe_action_t *
	create_group_pseudo_op(pe_resource_t group, const char action)
	{
	pe_action_t *op = custom_action(group, pcmk__op_key(group->id, action, 0),
	action, NULL, TRUE, TRUE, group->cluster);
	pe__set_action_flags(op, pcmk_action_pseudo\|pcmk_action_runnable);
	return op;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given group resource
	*
	* \param[in,out] rsc Group resource to create actions for
	*/
	void
	pcmk__group_create_actions(pe_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	pe_rsc_trace(rsc, "Creating actions for group %s", rsc->id);

	// Create actions for individual group members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->create_actions(member);
	}

	// Create pseudo-actions for group itself to serve as ordering points
	create_group_pseudo_op(rsc, PCMK_ACTION_START);
	create_group_pseudo_op(rsc, PCMK_ACTION_RUNNING);
	create_group_pseudo_op(rsc, PCMK_ACTION_STOP);
	create_group_pseudo_op(rsc, PCMK_ACTION_STOPPED);
	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROMOTABLE))) {
	create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTE);
	create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTED);
	create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTE);
	create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTED);
	}
	}

	// User data for member_internal_constraints()
	struct member_data {
	// These could be derived from member but this avoids some function calls
	bool ordered;
	bool colocated;
	bool promotable;

	pe_resource_t *last_active;
	pe_resource_t *previous_member;
	};

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group member
	*
	* \param[in,out] data Group member to create implicit constraints for
	* \param[in,out] user_data Member data (struct member_data *)
	*/
	static void
	member_internal_constraints(gpointer data, gpointer user_data)
	{
	pe_resource_t member = (pe_resource_t ) data;
	struct member_data member_data = (struct member_data ) user_data;

	// For ordering demote vs demote or stop vs stop
	uint32_t down_flags = pe_order_implies_first_printed;

	// For ordering demote vs demoted or stop vs stopped
	uint32_t post_down_flags = pe_order_implies_then_printed;

	// Create the individual member's implicit constraints
	member->cmds->internal_constraints(member);

	if (member_data->previous_member == NULL) {
	// This is first member
	if (member_data->ordered) {
	pe__set_order_flags(down_flags, pcmk__ar_ordered);
	- post_down_flags = pe_order_implies_then;
	+ post_down_flags = pcmk__ar_first_implies_then;
	}

	} else if (member_data->colocated) {
	uint32_t flags = pcmk__coloc_none;

	if (pcmk_is_set(member->flags, pcmk_rsc_critical)) {
	flags \|= pcmk__coloc_influence;
	}

	// Colocate this member with the previous one
	pcmk__new_colocation("#group-members", NULL, INFINITY, member,
	member_data->previous_member, NULL, NULL, flags);
	}

	if (member_data->promotable) {
	// Demote group -> demote member -> group is demoted
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_DEMOTE,
	member, PCMK_ACTION_DEMOTE, down_flags);
	pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
	member->parent, PCMK_ACTION_DEMOTED,
	post_down_flags);

	// Promote group -> promote member -> group is promoted
	pcmk__order_resource_actions(member, PCMK_ACTION_PROMOTE,
	member->parent, PCMK_ACTION_PROMOTED,
	pe_order_runnable_left
	- \|pe_order_implies_then
	+ \|pcmk__ar_first_implies_then
	\|pe_order_implies_then_printed);
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	pe_order_implies_first_printed);
	}

	// Stop group -> stop member -> group is stopped
	pcmk__order_stops(member->parent, member, down_flags);
	pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
	member->parent, PCMK_ACTION_STOPPED,
	post_down_flags);

	// Start group -> start member -> group is started
	pcmk__order_starts(member->parent, member, pe_order_implies_first_printed);
	pcmk__order_resource_actions(member, PCMK_ACTION_START,
	member->parent, PCMK_ACTION_RUNNING,
	pe_order_runnable_left
	- \|pe_order_implies_then
	+ \|pcmk__ar_first_implies_then
	\|pe_order_implies_then_printed);

	if (!member_data->ordered) {
	pcmk__order_starts(member->parent, member,
	- pe_order_implies_then
	+ pcmk__ar_first_implies_then
	\|pe_order_runnable_left
	\|pe_order_implies_first_printed);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	- pe_order_implies_then
	+ pcmk__ar_first_implies_then
	\|pe_order_runnable_left
	\|pe_order_implies_first_printed);
	}

	} else if (member_data->previous_member == NULL) {
	pcmk__order_starts(member->parent, member, pcmk__ar_none);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	pcmk__ar_none);
	}

	} else {
	// Order this member relative to the previous one

	pcmk__order_starts(member_data->previous_member, member,
	- pe_order_implies_then\|pe_order_runnable_left);
	+ pcmk__ar_first_implies_then
	+ \|pe_order_runnable_left);
	pcmk__order_stops(member, member_data->previous_member,
	pcmk__ar_ordered\|pe_order_restart);

	/* In unusual circumstances (such as adding a new member to the middle
	* of a group with unmanaged later members), this member may be active
	* while the previous (new) member is inactive. In this situation, the
	* usual restart orderings will be irrelevant, so we need to order this
	* member's stop before the previous member's start.
	*/
	if ((member->running_on != NULL)
	&& (member_data->previous_member->running_on == NULL)) {
	pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
	member_data->previous_member,
	PCMK_ACTION_START,
	pcmk__ar_then_implies_first
	\|pe_order_runnable_left);
	}

	if (member_data->promotable) {
	pcmk__order_resource_actions(member_data->previous_member,
	PCMK_ACTION_PROMOTE, member,
	PCMK_ACTION_PROMOTE,
	- pe_order_implies_then
	+ pcmk__ar_first_implies_then
	\|pe_order_runnable_left);
	pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
	member_data->previous_member,
	PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
	}
	}

	// Make sure partially active groups shut down in sequence
	if (member->running_on != NULL) {
	if (member_data->ordered && (member_data->previous_member != NULL)
	&& (member_data->previous_member->running_on == NULL)
	&& (member_data->last_active != NULL)
	&& (member_data->last_active->running_on != NULL)) {
	pcmk__order_stops(member, member_data->last_active,
	pcmk__ar_ordered);
	}
	member_data->last_active = member;
	}

	member_data->previous_member = member;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group resource
	*
	* \param[in,out] rsc Group resource to create implicit constraints for
	*/
	void
	pcmk__group_internal_constraints(pe_resource_t *rsc)
	{
	struct member_data member_data = { false, };
	const pe_resource_t *top = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	/* Order group pseudo-actions relative to each other for restarting:
	* stop group -> group is stopped -> start group -> group is started
	*/
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pe_order_runnable_left);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pe_order_runnable_left);

	top = pe__const_top_resource(rsc, false);

	member_data.ordered = pe__group_flag_is_set(rsc, pe__group_ordered);
	member_data.colocated = pe__group_flag_is_set(rsc, pe__group_colocated);
	member_data.promotable = pcmk_is_set(top->flags, pcmk_rsc_promotable);
	g_list_foreach(rsc->children, member_internal_constraints, &member_data);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint for a group with some other resource, apply the
	* score to the dependent's allowed node scores (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent group resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_group_with(pe_resource_t dependent, const pe_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	pe_resource_t *member = NULL;

	if (dependent->children == NULL) {
	return;
	}

	pe_rsc_trace(primary, "Processing %s (group %s with %s) for dependent",
	colocation->id, dependent->id, primary->id);

	if (pe__group_flag_is_set(dependent, pe__group_colocated)) {
	// Colocate first member (internal colocations will handle the rest)
	member = (pe_resource_t *) dependent->children->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation between "
	"non-colocated group and %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate each member individually
	for (GList *iter = dependent->children; iter != NULL; iter = iter->next) {
	member = (pe_resource_t *) iter->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint for some other resource with a group, apply the
	* score to the dependent's allowed node scores (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary group resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_with_group(pe_resource_t dependent, const pe_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	const pe_resource_t *member = NULL;

	pe_rsc_trace(primary,
	"Processing colocation %s (%s with group %s) for primary",
	colocation->id, dependent->id, primary->id);

	if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	return;
	}

	if (pe__group_flag_is_set(primary, pe__group_colocated)) {

	if (colocation->score >= INFINITY) {
	/* For mandatory colocations, the entire group must be assignable
	* (and in the specified role if any), so apply the colocation based
	* on the last member.
	*/
	member = pe__last_group_member(primary);
	} else if (primary->children != NULL) {
	/* For optional colocations, whether the group is partially or fully
	* up doesn't matter, so apply the colocation based on the first
	* member.
	*/
	member = (pe_resource_t *) primary->children->data;
	}
	if (member == NULL) {
	return; // Nothing to colocate with
	}

	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation with"
	" non-colocated group %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate dependent with each member individually
	for (const GList *iter = primary->children; iter != NULL;
	iter = iter->next) {
	member = iter->data;
	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__group_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	colocate_group_with(dependent, primary, colocation);

	} else {
	// Method should only be called for primitive dependents
	CRM_ASSERT(dependent->variant == pcmk_rsc_variant_primitive);

	colocate_with_group(dependent, primary, colocation);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given group resource action
	*
	* \param[in,out] action Group action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__group_action_flags(pe_action_t action, const pe_node_t node)
	{
	// Default flags for a group action
	uint32_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	CRM_ASSERT(action != NULL);

	// Update flags considering each member's own flags for same action
	for (GList *iter = action->rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	// Check whether member has the same action
	enum action_tasks task = get_complex_task(member, action->task);
	const char *task_s = task2text(task);
	pe_action_t *member_action = find_first_action(member->actions, NULL,
	task_s, node);

	if (member_action != NULL) {
	uint32_t member_flags = member->cmds->action_flags(member_action,
	node);

	// Group action is mandatory if any member action is
	if (pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(member_flags, pcmk_action_optional)) {
	pe_rsc_trace(action->rsc, "%s is mandatory because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_optional);
	pe__clear_action_flags(action, pcmk_action_optional);
	}

	// Group action is unrunnable if any member action is
	if (!pcmk__str_eq(task_s, action->task, pcmk__str_none)
	&& pcmk_is_set(flags, pcmk_action_runnable)
	&& !pcmk_is_set(member_flags, pcmk_action_runnable)) {

	pe_rsc_trace(action->rsc, "%s is unrunnable because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_runnable);
	pe__clear_action_flags(action, pcmk_action_runnable);
	}

	/* Group (pseudo-)actions other than stop or demote are unrunnable
	* unless every member will do it.
	*/
	} else if ((task != pcmk_action_stop) && (task != pcmk_action_demote)) {
	pe_rsc_trace(action->rsc,
	"%s is not runnable because %s will not %s",
	action->uuid, member->id, task_s);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_runnable);
	}
	}

	return flags;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__group_update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;

	// Group method can be called only on behalf of "then" action
	CRM_ASSERT((first != NULL) && (then != NULL) && (then->rsc != NULL)
	&& (data_set != NULL));

	// Update the actions for the group itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags, filter,
	type, data_set);

	// Update the actions for each group member
	for (GList *iter = then->rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	pe_action_t *member_action = find_first_action(member->actions, NULL,
	then->task, node);

	if (member_action != NULL) {
	changed \|= member->cmds->update_ordered_actions(first,
	member_action, node,
	flags, filter, type,
	data_set);
	}
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a group's allowed node scores
	*
	* \param[in,out] rsc Group resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__group_apply_location(pe_resource_t rsc, pe__location_t location)
	{
	GList *node_list_orig = NULL;
	GList *node_list_copy = NULL;
	bool reset_scores = true;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (location != NULL));

	node_list_orig = location->node_list_rh;
	node_list_copy = pcmk__copy_node_list(node_list_orig, true);
	reset_scores = pe__group_flag_is_set(rsc, pe__group_colocated);

	// Apply the constraint for the group itself (updates node scores)
	pcmk__apply_location(rsc, location);

	// Apply the constraint for each member
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->apply_location(member, location);

	if (reset_scores) {
	/* The first member of colocated groups needs to use the original
	* node scores, but subsequent members should work on a copy, since
	* the first member's scores already incorporate theirs.
	*/
	reset_scores = false;
	location->node_list_rh = node_list_copy;
	}
	}

	location->node_list_rh = node_list_orig;
	g_list_free_full(node_list_copy, free);
	}

	// Group implementation of resource_alloc_functions_t:colocated_resources()
	GList *
	pcmk__group_colocated_resources(const pe_resource_t *rsc,
	const pe_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const pe_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if (pe__group_flag_is_set(rsc, pe__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	/* This group has colocated members and/or is cloned -- either way,
	* add every child's colocated resources to the list. The first and last
	* members will include the group's own colocations.
	*/
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	member = (const pe_resource_t *) iter->data;
	colocated_rscs = member->cmds->colocated_resources(member, orig_rsc,
	colocated_rscs);
	}

	} else if (rsc->children != NULL) {
	/* This group's members are not colocated, and the group is not cloned,
	* so just add the group's own colocations to the list.
	*/
	colocated_rscs = pcmk__colocated_resources(rsc, orig_rsc,
	colocated_rscs);
	}

	return colocated_rscs;
	}

	// Group implementation of resource_alloc_functions_t:with_this_colocations()
	void
	pcmk__with_group_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)

	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (list != NULL));

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* "With this" colocations are needed only for the group itself and for its
	* last member. (Previous members will chain via the group internal
	* colocations.)
	*/
	if ((orig_rsc != rsc) && (orig_rsc != pe__last_group_member(rsc))) {
	return;
	}

	pe_rsc_trace(rsc, "Adding 'with %s' colocations to list for %s",
	rsc->id, orig_rsc->id);

	// Add the group's own colocations
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);

	// If cloned, add any relevant colocations with the clone
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc,
	list);
	}

	if (!pe__group_flag_is_set(rsc, pe__group_colocated)) {
	// @COMPAT Non-colocated groups are deprecated
	return;
	}

	// Add explicit colocations with the group's (other) children
	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pe_resource_t *member = iter->data;

	if (member != orig_rsc) {
	member->cmds->with_this_colocations(member, orig_rsc, list);
	}
	}
	}

	// Group implementation of resource_alloc_functions_t:this_with_colocations()
	void
	pcmk__group_with_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	const pe_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (list != NULL));

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* "This with" colocations are normally needed only for the group itself and
	* for its first member.
	*/
	if ((rsc == orig_rsc)
	\|\| (orig_rsc == (const pe_resource_t *) rsc->children->data)) {
	pe_rsc_trace(rsc, "Adding '%s with' colocations to list for %s",
	rsc->id, orig_rsc->id);

	// Add the group's own colocations
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);

	// If cloned, add any relevant colocations involving the clone
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc,
	list);
	}

	if (!pe__group_flag_is_set(rsc, pe__group_colocated)) {
	// @COMPAT Non-colocated groups are deprecated
	return;
	}

	// Add explicit colocations involving the group's (other) children
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {
	member = iter->data;
	if (member != orig_rsc) {
	member->cmds->this_with_colocations(member, orig_rsc, list);
	}
	}
	return;
	}

	/* Later group members honor the group's colocations indirectly, due to the
	* internal group colocations that chain everything from the first member.
	* However, if an earlier group member is unmanaged, this chaining will not
	* happen, so the group's mandatory colocations must be explicitly added.
	*/
	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	member = iter->data;
	if (orig_rsc == member) {
	break; // We've seen all earlier members, and none are unmanaged
	}

	if (!pcmk_is_set(member->flags, pcmk_rsc_managed)) {
	crm_trace("Adding mandatory '%s with' colocations to list for "
	"member %s because earlier member %s is unmanaged",
	rsc->id, orig_rsc->id, member->id);
	for (const GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
	cons_iter = cons_iter->next) {
	const pcmk__colocation_t *colocation = NULL;

	colocation = (const pcmk__colocation_t *) cons_iter->data;
	if (colocation->score == INFINITY) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	// @TODO Add mandatory (or all?) clone constraints if cloned
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Group resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to \c NULL
	* to copy allowed nodes from \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores will
	* not be added, and \p *nodes must be \c NULL as
	* well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
	* the \c pcmk__coloc_select_this_with flag are used together (and only by
	* \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	* \note This is the group implementation of
	* \c resource_alloc_functions_t:add_colocated_node_scores().
	*/
	void
	pcmk__group_add_colocated_node_scores(pe_resource_t *source_rsc,
	const pe_resource_t *target_rsc,
	const char log_id, GHashTable *nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags)
	{
	pe_resource_t *member = NULL;

	CRM_ASSERT((source_rsc != NULL)
	&& (source_rsc->variant == pcmk_rsc_variant_group)
	&& (nodes != NULL)
	&& ((colocation != NULL)
	\|\| ((target_rsc == NULL) && (*nodes == NULL))));

	if (log_id == NULL) {
	log_id = source_rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(source_rsc->flags, pcmk_rsc_updating_nodes)) {
	pe_rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
	log_id, source_rsc->id);
	return;
	}
	pe__set_resource_flags(source_rsc, pcmk_rsc_updating_nodes);

	// Ignore empty groups (only possible with schema validation disabled)
	if (source_rsc->children == NULL) {
	return;
	}

	/* Refer the operation to the first or last member as appropriate.
	*
	* cmp_resources() is the only caller that passes a NULL nodes table,
	* and is also the only caller using pcmk__coloc_select_this_with.
	* For "this with" colocations, the last member will recursively incorporate
	* all the other members' "this with" colocations via the internal group
	* colocations (and via the first member, the group's own colocations).
	*
	* For "with this" colocations, the first member works similarly.
	*/
	if (*nodes == NULL) {
	member = pe__last_group_member(source_rsc);
	} else {
	member = source_rsc->children->data;
	}
	pe_rsc_trace(source_rsc, "%s: Merging scores from group %s using member %s "
	"(at %.6f)", log_id, source_rsc->id, member->id, factor);
	member->cmds->add_colocated_node_scores(member, target_rsc, log_id, nodes,
	colocation, factor, flags);
	pe__clear_resource_flags(source_rsc, pcmk_rsc_updating_nodes);
	}

	// Group implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__group_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	pe_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding group %s as colocated utilization",
	orig_rsc->id, rsc->id);
	if (pe__group_flag_is_set(rsc, pe__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	// Every group member will be on same node, so sum all members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	member = (pe_resource_t *) iter->data;

	if (pcmk_is_set(member->flags, pcmk_rsc_unassigned)
	&& (g_list_find(all_rscs, member) == NULL)) {
	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}

	} else if (rsc->children != NULL) {
	// Just add first member's utilization
	member = (pe_resource_t *) rsc->children->data;
	if ((member != NULL)
	&& pcmk_is_set(member->flags, pcmk_rsc_unassigned)
	&& (g_list_find(all_rscs, member) == NULL)) {

	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}
	}

	// Group implementation of resource_alloc_functions_t:shutdown_lock()
	void
	pcmk__group_shutdown_lock(pe_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t member = (pe_resource_t ) iter->data;

	member->cmds->shutdown_lock(member);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_instances.c b/lib/pacemaker/pcmk_sched_instances.c
	index e937c5c1df..9071ada3f2 100644
	--- a/lib/pacemaker/pcmk_sched_instances.c
	+++ b/lib/pacemaker/pcmk_sched_instances.c
	@@ -1,1675 +1,1675 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	/* This file is intended for code usable with both clone instances and bundle
	* replica containers.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Check whether a node is allowed to run an instance
	*
	* \param[in] instance Clone instance or bundle container to check
	* \param[in] node Node to check
	* \param[in] max_per_node Maximum number of instances allowed to run on a node
	*
	* \return true if \p node is allowed to run \p instance, otherwise false
	*/
	static bool
	can_run_instance(const pe_resource_t instance, const pe_node_t node,
	int max_per_node)
	{
	pe_node_t *allowed_node = NULL;

	if (pcmk_is_set(instance->flags, pcmk_rsc_removed)) {
	pe_rsc_trace(instance, "%s cannot run on %s: orphaned",
	instance->id, pe__node_name(node));
	return false;
	}

	if (!pcmk__node_available(node, false, false)) {
	pe_rsc_trace(instance,
	"%s cannot run on %s: node cannot run resources",
	instance->id, pe__node_name(node));
	return false;
	}

	allowed_node = pcmk__top_allowed_node(instance, node);
	if (allowed_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed",
	instance->id, pe__node_name(node));
	return false;
	}

	if (allowed_node->weight < 0) {
	pe_rsc_trace(instance, "%s cannot run on %s: parent score is %s there",
	instance->id, pe__node_name(node),
	pcmk_readable_score(allowed_node->weight));
	return false;
	}

	if (allowed_node->count >= max_per_node) {
	pe_rsc_trace(instance,
	"%s cannot run on %s: node already has %d instance%s",
	instance->id, pe__node_name(node), max_per_node,
	pcmk__plural_s(max_per_node));
	return false;
	}

	pe_rsc_trace(instance, "%s can run on %s (%d already running)",
	instance->id, pe__node_name(node), allowed_node->count);
	return true;
	}

	/*!
	* \internal
	* \brief Ban a clone instance or bundle replica from unavailable allowed nodes
	*
	* \param[in,out] instance Clone instance or bundle replica to ban
	* \param[in] max_per_node Maximum instances allowed to run on a node
	*/
	static void
	ban_unavailable_allowed_nodes(pe_resource_t *instance, int max_per_node)
	{
	if (instance->allowed_nodes != NULL) {
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, instance->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!can_run_instance(instance, node, max_per_node)) {
	pe_rsc_trace(instance, "Banning %s from unavailable node %s",
	instance->id, pe__node_name(node));
	node->weight = -INFINITY;
	for (GList *child_iter = instance->children;
	child_iter != NULL; child_iter = child_iter->next) {
	pe_resource_t child = (pe_resource_t ) child_iter->data;
	pe_node_t *child_node = NULL;

	child_node = g_hash_table_lookup(child->allowed_nodes,
	node->details->id);
	if (child_node != NULL) {
	pe_rsc_trace(instance,
	"Banning %s child %s "
	"from unavailable node %s",
	instance->id, child->id,
	pe__node_name(node));
	child_node->weight = -INFINITY;
	}
	}
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create a hash table with a single node in it
	*
	* \param[in] node Node to copy into new table
	*
	* \return Newly created hash table containing a copy of \p node
	* \note The caller is responsible for freeing the result with
	* g_hash_table_destroy().
	*/
	static GHashTable *
	new_node_table(pe_node_t *node)
	{
	GHashTable *table = pcmk__strkey_table(NULL, free);

	node = pe__copy_node(node);
	g_hash_table_insert(table, (gpointer) node->details->id, node);
	return table;
	}

	/*!
	* \internal
	* \brief Apply a resource's parent's colocation scores to a node table
	*
	* \param[in] rsc Resource whose colocations should be applied
	* \param[in,out] nodes Node table to apply colocations to
	*/
	static void
	apply_parent_colocations(const pe_resource_t rsc, GHashTable *nodes)
	{
	GList *colocations = pcmk__this_with_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pe_resource_t *other = colocation->primary;
	float factor = colocation->score / (float) INFINITY;

	other->cmds->add_colocated_node_scores(other, rsc, rsc->id, nodes,
	colocation, factor,
	pcmk__coloc_select_default);
	}
	g_list_free(colocations);
	colocations = pcmk__with_this_colocations(rsc);

	for (const GList *iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *colocation = iter->data;
	pe_resource_t *other = colocation->dependent;
	float factor = colocation->score / (float) INFINITY;

	if (!pcmk__colocation_has_influence(colocation, rsc)) {
	continue;
	}
	other->cmds->add_colocated_node_scores(other, rsc, rsc->id, nodes,
	colocation, factor,
	pcmk__coloc_select_nonnegative);
	}
	g_list_free(colocations);
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances based on colocation scores
	*
	* Determine the relative order in which two clone or bundle instances should be
	* assigned to nodes, considering the scores of colocation constraints directly
	* or indirectly involving them.
	*
	* \param[in] instance1 First instance to compare
	* \param[in] instance2 Second instance to compare
	*
	* \return A negative number if \p instance1 should be assigned first,
	* a positive number if \p instance2 should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	static int
	cmp_instance_by_colocation(const pe_resource_t *instance1,
	const pe_resource_t *instance2)
	{
	int rc = 0;
	pe_node_t *node1 = NULL;
	pe_node_t *node2 = NULL;
	pe_node_t *current_node1 = pe__current_node(instance1);
	pe_node_t *current_node2 = pe__current_node(instance2);
	GHashTable *colocated_scores1 = NULL;
	GHashTable *colocated_scores2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance1->parent != NULL)
	&& (instance2 != NULL) && (instance2->parent != NULL)
	&& (current_node1 != NULL) && (current_node2 != NULL));

	// Create node tables initialized with each node
	colocated_scores1 = new_node_table(current_node1);
	colocated_scores2 = new_node_table(current_node2);

	// Apply parental colocations
	apply_parent_colocations(instance1, &colocated_scores1);
	apply_parent_colocations(instance2, &colocated_scores2);

	// Find original nodes again, with scores updated for colocations
	node1 = g_hash_table_lookup(colocated_scores1, current_node1->details->id);
	node2 = g_hash_table_lookup(colocated_scores2, current_node2->details->id);

	// Compare nodes by updated scores
	if (node1->weight < node2->weight) {
	crm_trace("Assign %s (%d on %s) after %s (%d on %s)",
	instance1->id, node1->weight, pe__node_name(node1),
	instance2->id, node2->weight, pe__node_name(node2));
	rc = 1;

	} else if (node1->weight > node2->weight) {
	crm_trace("Assign %s (%d on %s) before %s (%d on %s)",
	instance1->id, node1->weight, pe__node_name(node1),
	instance2->id, node2->weight, pe__node_name(node2));
	rc = -1;
	}

	g_hash_table_destroy(colocated_scores1);
	g_hash_table_destroy(colocated_scores2);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its children are failed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc or any of its children are failed, otherwise false
	*/
	static bool
	did_fail(const pe_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (did_fail((const pe_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in,out] node Node to check (will be set NULL if not allowed)
	*
	* \return true if *node is either NULL or allowed for \p rsc, otherwise false
	*/
	static bool
	node_is_allowed(const pe_resource_t rsc, pe_node_t *node)
	{
	if (*node != NULL) {
	pe_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
	(*node)->details->id);

	if ((allowed == NULL) \|\| (allowed->weight < 0)) {
	pe_rsc_trace(rsc, "%s: current location (%s) is unavailable",
	rsc->id, pe__node_name(*node));
	*node = NULL;
	return false;
	}
	}
	return true;
	}

	/*!
	* \internal
	* \brief Compare two clone or bundle instances' instance numbers
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a's instance number is lower,
	* a positive number if \p b's instance number is lower,
	* or 0 if their instance numbers are the same
	*/
	gint
	pcmk__cmp_instance_number(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t instance1 = (const pe_resource_t ) a;
	const pe_resource_t instance2 = (const pe_resource_t ) b;
	char *div1 = NULL;
	char *div2 = NULL;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	// Clone numbers are after a colon, bundle numbers after a dash
	div1 = strrchr(instance1->id, ':');
	if (div1 == NULL) {
	div1 = strrchr(instance1->id, '-');
	}
	div2 = strrchr(instance2->id, ':');
	if (div2 == NULL) {
	div2 = strrchr(instance2->id, '-');
	}
	CRM_ASSERT((div1 != NULL) && (div2 != NULL));

	return (gint) (strtol(div1 + 1, NULL, 10) - strtol(div2 + 1, NULL, 10));
	}

	/*!
	* \internal
	* \brief Compare clone or bundle instances according to assignment order
	*
	* Compare two clone or bundle instances according to the order they should be
	* assigned to nodes, preferring (in order):
	*
	* - Active instance that is less multiply active
	* - Instance that is not active on a disallowed node
	* - Instance with higher configured priority
	* - Active instance whose current node can run resources
	* - Active instance whose parent is allowed on current node
	* - Active instance whose current node has fewer other instances
	* - Active instance
	* - Instance that isn't failed
	* - Instance whose colocations result in higher score on current node
	* - Instance with lower ID in lexicographic order
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a should be assigned first,
	* a positive number if \p b should be assigned first,
	* or 0 if assignment order doesn't matter
	*/
	gint
	pcmk__cmp_instance(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	pe_node_t *node1 = NULL;
	pe_node_t *node2 = NULL;
	unsigned int nnodes1 = 0;
	unsigned int nnodes2 = 0;

	bool can1 = true;
	bool can2 = true;

	const pe_resource_t instance1 = (const pe_resource_t ) a;
	const pe_resource_t instance2 = (const pe_resource_t ) b;

	CRM_ASSERT((instance1 != NULL) && (instance2 != NULL));

	node1 = instance1->fns->active_node(instance1, &nnodes1, NULL);
	node2 = instance2->fns->active_node(instance2, &nnodes2, NULL);

	/* If both instances are running and at least one is multiply
	* active, prefer instance that's running on fewer nodes.
	*/
	if ((nnodes1 > 0) && (nnodes2 > 0)) {
	if (nnodes1 < nnodes2) {
	crm_trace("Assign %s (active on %d) before %s (active on %d): "
	"less multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return -1;

	} else if (nnodes1 > nnodes2) {
	crm_trace("Assign %s (active on %d) after %s (active on %d): "
	"more multiply active",
	instance1->id, nnodes1, instance2->id, nnodes2);
	return 1;
	}
	}

	/* An instance that is either inactive or active on an allowed node is
	* preferred over an instance that is active on a no-longer-allowed node.
	*/
	can1 = node_is_allowed(instance1, &node1);
	can2 = node_is_allowed(instance2, &node2);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: not active on a disallowed node",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: active on a disallowed node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher configured priority
	if (instance1->priority > instance2->priority) {
	crm_trace("Assign %s before %s: priority (%d > %d)",
	instance1->id, instance2->id,
	instance1->priority, instance2->priority);
	return -1;

	} else if (instance1->priority < instance2->priority) {
	crm_trace("Assign %s after %s: priority (%d < %d)",
	instance1->id, instance2->id,
	instance1->priority, instance2->priority);
	return 1;
	}

	// Prefer active instance
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: inactive",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: active", instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: active", instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node can run resources
	can1 = pcmk__node_available(node1, false, false);
	can2 = pcmk__node_available(node2, false, false);
	if (can1 && !can2) {
	crm_trace("Assign %s before %s: current node can run resources",
	instance1->id, instance2->id);
	return -1;

	} else if (!can1 && can2) {
	crm_trace("Assign %s after %s: current node can't run resources",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance whose parent is allowed to run on instance's current node
	node1 = pcmk__top_allowed_node(instance1, node1);
	node2 = pcmk__top_allowed_node(instance2, node2);
	if ((node1 == NULL) && (node2 == NULL)) {
	crm_trace("No assignment preference for %s vs. %s: "
	"parent not allowed on either instance's current node",
	instance1->id, instance2->id);
	return 0;

	} else if (node1 == NULL) {
	crm_trace("Assign %s after %s: parent not allowed on current node",
	instance1->id, instance2->id);
	return 1;

	} else if (node2 == NULL) {
	crm_trace("Assign %s before %s: parent allowed on current node",
	instance1->id, instance2->id);
	return -1;
	}

	// Prefer instance whose current node is running fewer other instances
	if (node1->count < node2->count) {
	crm_trace("Assign %s before %s: fewer active instances on current node",
	instance1->id, instance2->id);
	return -1;

	} else if (node1->count > node2->count) {
	crm_trace("Assign %s after %s: more active instances on current node",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance that isn't failed
	can1 = did_fail(instance1);
	can2 = did_fail(instance2);
	if (!can1 && can2) {
	crm_trace("Assign %s before %s: not failed",
	instance1->id, instance2->id);
	return -1;
	} else if (can1 && !can2) {
	crm_trace("Assign %s after %s: failed",
	instance1->id, instance2->id);
	return 1;
	}

	// Prefer instance with higher cumulative colocation score on current node
	rc = cmp_instance_by_colocation(instance1, instance2);
	if (rc != 0) {
	return rc;
	}

	// Prefer instance with lower instance number
	rc = pcmk__cmp_instance_number(instance1, instance2);
	if (rc < 0) {
	crm_trace("Assign %s before %s: instance number",
	instance1->id, instance2->id);
	} else if (rc > 0) {
	crm_trace("Assign %s after %s: instance number",
	instance1->id, instance2->id);
	} else {
	crm_trace("No assignment preference for %s vs. %s",
	instance1->id, instance2->id);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Increment the parent's instance count after assigning an instance
	*
	* An instance's parent tracks how many instances have been assigned to each
	* node via its pe_node_t:count member. After assigning an instance to a node,
	* find the corresponding node in the parent's allowed table and increment it.
	*
	* \param[in,out] instance Instance whose parent to update
	* \param[in] assigned_to Node to which the instance was assigned
	*/
	static void
	increment_parent_count(pe_resource_t instance, const pe_node_t assigned_to)
	{
	pe_node_t *allowed = NULL;

	if (assigned_to == NULL) {
	return;
	}
	allowed = pcmk__top_allowed_node(instance, assigned_to);

	if (allowed == NULL) {
	/* The instance is allowed on the node, but its parent isn't. This
	* shouldn't be possible if the resource is managed, and we won't be
	* able to limit the number of instances assigned to the node.
	*/
	CRM_LOG_ASSERT(!pcmk_is_set(instance->flags, pcmk_rsc_managed));

	} else {
	allowed->count++;
	}
	}

	/*!
	* \internal
	* \brief Assign an instance to a node
	*
	* \param[in,out] instance Clone instance or bundle replica container
	* \param[in] prefer If not NULL, attempt early assignment to this
	* node, if still the best choice; otherwise,
	* perform final assignment
	* \param[in] max_per_node Assign at most this many instances to one node
	*
	* \return Node to which \p instance is assigned
	*/
	static const pe_node_t *
	assign_instance(pe_resource_t instance, const pe_node_t prefer,
	int max_per_node)
	{
	pe_node_t *chosen = NULL;

	pe_rsc_trace(instance, "Assigning %s (preferring %s)", instance->id,
	((prefer == NULL)? "no node" : prefer->details->uname));

	if (pcmk_is_set(instance->flags, pcmk_rsc_assigning)) {
	pe_rsc_debug(instance,
	"Assignment loop detected involving %s colocations",
	instance->id);
	return NULL;
	}
	ban_unavailable_allowed_nodes(instance, max_per_node);

	// Failed early assignments are reversible (stop_if_fail=false)
	chosen = instance->cmds->assign(instance, prefer, (prefer == NULL));
	increment_parent_count(instance, chosen);
	return chosen;
	}

	/*!
	* \internal
	* \brief Try to assign an instance to its current node early
	*
	* \param[in] rsc Clone or bundle being assigned (for logs only)
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] current Instance's current node
	* \param[in] max_per_node Maximum number of instances per node
	* \param[in] available Number of instances still available for assignment
	*
	* \return \c true if \p instance was successfully assigned to its current node,
	* or \c false otherwise
	*/
	static bool
	assign_instance_early(const pe_resource_t rsc, pe_resource_t instance,
	const pe_node_t *current, int max_per_node, int available)
	{
	const pe_node_t *chosen = NULL;
	int reserved = 0;

	pe_resource_t *parent = instance->parent;
	GHashTable *allowed_orig = NULL;
	GHashTable *allowed_orig_parent = parent->allowed_nodes;

	const pe_node_t *allowed_node = g_hash_table_lookup(instance->allowed_nodes,
	current->details->id);

	pe_rsc_trace(instance, "Trying to assign %s to its current node %s",
	instance->id, pe__node_name(current));

	if (!pcmk__node_available(allowed_node, true, false)) {
	pe_rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pe__node_name(current));
	return false;
	}

	/* On each iteration, if instance gets assigned to a node other than its
	* current one, we reserve one instance for the chosen node, unassign
	* instance, restore instance's original node tables, and try again. This
	* way, instances are proportionally assigned to nodes based on preferences,
	* but shuffling of specific instances is minimized. If a node will be
	* assigned instances at all, it preferentially receives instances that are
	* currently active there.
	*
	* parent->allowed_nodes tracks the number of instances assigned to each
	* node. If a node already has max_per_node instances assigned,
	* ban_unavailable_allowed_nodes() marks it as unavailable.
	*
	* In the end, we restore the original parent->allowed_nodes to undo the
	* changes to counts during tentative assignments. If we successfully
	* assigned instance to its current node, we increment that node's counter.
	*/

	// Back up the allowed node tables of instance and its children recursively
	pcmk__copy_node_tables(instance, &allowed_orig);

	// Update instances-per-node counts in a scratch table
	parent->allowed_nodes = pcmk__copy_node_table(parent->allowed_nodes);

	while (reserved < available) {
	chosen = assign_instance(instance, current, max_per_node);

	if (pe__same_node(chosen, current)) {
	// Successfully assigned to current node
	break;
	}

	// Assignment updates scores, so restore to original state
	pe_rsc_debug(instance, "Rolling back node scores for %s", instance->id);
	pcmk__restore_node_tables(instance, allowed_orig);

	if (chosen == NULL) {
	// Assignment failed, so give up
	pe_rsc_info(instance,
	"Not assigning %s to current node %s: unavailable",
	instance->id, pe__node_name(current));
	pe__set_resource_flags(instance, pcmk_rsc_unassigned);
	break;
	}

	// We prefer more strongly to assign an instance to the chosen node
	pe_rsc_debug(instance,
	"Not assigning %s to current node %s: %s is better",
	instance->id, pe__node_name(current),
	pe__node_name(chosen));

	// Reserve one instance for the chosen node and try again
	if (++reserved >= available) {
	pe_rsc_info(instance,
	"Not assigning %s to current node %s: "
	"other assignments are more important",
	instance->id, pe__node_name(current));

	} else {
	pe_rsc_debug(instance,
	"Reserved an instance of %s for %s. Retrying "
	"assignment of %s to %s",
	rsc->id, pe__node_name(chosen), instance->id,
	pe__node_name(current));
	}

	// Clear this assignment (frees chosen); leave instance counts in parent
	pcmk__unassign_resource(instance);
	chosen = NULL;
	}

	g_hash_table_destroy(allowed_orig);

	// Restore original instances-per-node counts
	g_hash_table_destroy(parent->allowed_nodes);
	parent->allowed_nodes = allowed_orig_parent;

	if (chosen == NULL) {
	// Couldn't assign instance to current node
	return false;
	}
	pe_rsc_trace(instance, "Assigned %s to current node %s",
	instance->id, pe__node_name(current));
	increment_parent_count(instance, chosen);
	return true;
	}

	/*!
	* \internal
	* \brief Reset the node counts of a resource's allowed nodes to zero
	*
	* \param[in,out] rsc Resource to reset
	*
	* \return Number of nodes that are available to run resources
	*/
	static unsigned int
	reset_allowed_node_counts(pe_resource_t *rsc)
	{
	unsigned int available_nodes = 0;
	pe_node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	node->count = 0;
	if (pcmk__node_available(node, false, false)) {
	available_nodes++;
	}
	}
	return available_nodes;
	}

	/*!
	* \internal
	* \brief Check whether an instance has a preferred node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] optimal_per_node Optimal number of instances per node
	*
	* \return Instance's current node if still available, otherwise NULL
	*/
	static const pe_node_t *
	preferred_node(const pe_resource_t *instance, int optimal_per_node)
	{
	const pe_node_t *node = NULL;
	const pe_node_t *parent_node = NULL;

	// Check whether instance is active, healthy, and not yet assigned
	if ((instance->running_on == NULL)
	\|\| !pcmk_is_set(instance->flags, pcmk_rsc_unassigned)
	\|\| pcmk_is_set(instance->flags, pcmk_rsc_failed)) {
	return NULL;
	}

	// Check whether instance's current node can run resources
	node = pe__current_node(instance);
	if (!pcmk__node_available(node, true, false)) {
	pe_rsc_trace(instance, "Not assigning %s to %s early (unavailable)",
	instance->id, pe__node_name(node));
	return NULL;
	}

	// Check whether node already has optimal number of instances assigned
	parent_node = pcmk__top_allowed_node(instance, node);
	if ((parent_node != NULL) && (parent_node->count >= optimal_per_node)) {
	pe_rsc_trace(instance,
	"Not assigning %s to %s early "
	"(optimal instances already assigned)",
	instance->id, pe__node_name(node));
	return NULL;
	}

	return node;
	}

	/*!
	* \internal
	* \brief Assign collective instances to nodes
	*
	* \param[in,out] collective Clone or bundle resource being assigned
	* \param[in,out] instances List of clone instances or bundle containers
	* \param[in] max_total Maximum instances to assign in total
	* \param[in] max_per_node Maximum instances to assign to any one node
	*/
	void
	pcmk__assign_instances(pe_resource_t collective, GList instances,
	int max_total, int max_per_node)
	{
	// Reuse node count to track number of assigned instances
	unsigned int available_nodes = reset_allowed_node_counts(collective);

	int optimal_per_node = 0;
	int assigned = 0;
	GList *iter = NULL;
	pe_resource_t *instance = NULL;
	const pe_node_t *current = NULL;

	if (available_nodes > 0) {
	optimal_per_node = max_total / available_nodes;
	}
	if (optimal_per_node < 1) {
	optimal_per_node = 1;
	}

	pe_rsc_debug(collective,
	"Assigning up to %d %s instance%s to up to %u node%s "
	"(at most %d per host, %d optimal)",
	max_total, collective->id, pcmk__plural_s(max_total),
	available_nodes, pcmk__plural_s(available_nodes),
	max_per_node, optimal_per_node);

	// Assign as many instances as possible to their current location
	for (iter = instances; (iter != NULL) && (assigned < max_total);
	iter = iter->next) {
	int available = max_total - assigned;

	instance = iter->data;
	if (!pcmk_is_set(instance->flags, pcmk_rsc_unassigned)) {
	continue; // Already assigned
	}

	current = preferred_node(instance, optimal_per_node);
	if ((current != NULL)
	&& assign_instance_early(collective, instance, current,
	max_per_node, available)) {
	assigned++;
	}
	}

	pe_rsc_trace(collective, "Assigned %d of %d instance%s to current node",
	assigned, max_total, pcmk__plural_s(max_total));

	for (iter = instances; iter != NULL; iter = iter->next) {
	instance = (pe_resource_t *) iter->data;

	if (!pcmk_is_set(instance->flags, pcmk_rsc_unassigned)) {
	continue; // Already assigned
	}

	if (instance->running_on != NULL) {
	current = pe__current_node(instance);
	if (pcmk__top_allowed_node(instance, current) == NULL) {
	const char *unmanaged = "";

	if (!pcmk_is_set(instance->flags, pcmk_rsc_managed)) {
	unmanaged = "Unmanaged resource ";
	}
	crm_notice("%s%s is running on %s which is no longer allowed",
	unmanaged, instance->id, pe__node_name(current));
	}
	}

	if (assigned >= max_total) {
	pe_rsc_debug(collective,
	"Not assigning %s because maximum %d instances "
	"already assigned",
	instance->id, max_total);
	resource_location(instance, NULL, -INFINITY,
	"collective_limit_reached", collective->cluster);

	} else if (assign_instance(instance, NULL, max_per_node) != NULL) {
	assigned++;
	}
	}

	pe_rsc_debug(collective, "Assigned %d of %d possible instance%s of %s",
	assigned, max_total, pcmk__plural_s(max_total),
	collective->id);
	}

	enum instance_state {
	instance_starting = (1 << 0),
	instance_stopping = (1 << 1),

	/* This indicates that some instance is restarting. It's not the same as
	* instance_starting\|instance_stopping, which would indicate that some
	* instance is starting, and some instance (not necessarily the same one) is
	* stopping.
	*/
	instance_restarting = (1 << 2),

	instance_active = (1 << 3),

	instance_all = instance_starting\|instance_stopping
	\|instance_restarting\|instance_active,
	};

	/*!
	* \internal
	* \brief Check whether an instance is active, starting, and/or stopping
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in,out] state Whether any instance is starting, stopping, etc.
	*/
	static void
	check_instance_state(const pe_resource_t instance, uint32_t state)
	{
	const GList *iter = NULL;
	uint32_t instance_state = 0; // State of just this instance

	// No need to check further if all conditions have already been detected
	if (pcmk_all_flags_set(*state, instance_all)) {
	return;
	}

	// If instance is a collective (a cloned group), check its children instead
	if (instance->variant > pcmk_rsc_variant_primitive) {
	for (iter = instance->children;
	(iter != NULL) && !pcmk_all_flags_set(*state, instance_all);
	iter = iter->next) {
	check_instance_state((const pe_resource_t *) iter->data, state);
	}
	return;
	}

	// If we get here, instance is a primitive

	if (instance->running_on != NULL) {
	instance_state \|= instance_active;
	}

	// Check each of the instance's actions for runnable start or stop
	for (iter = instance->actions;
	(iter != NULL) && !pcmk_all_flags_set(instance_state,
	instance_starting
	\|instance_stopping);
	iter = iter->next) {

	const pe_action_t action = (const pe_action_t ) iter->data;
	const bool optional = pcmk_is_set(action->flags, pcmk_action_optional);

	if (pcmk__str_eq(PCMK_ACTION_START, action->task, pcmk__str_none)) {
	if (!optional
	&& pcmk_is_set(action->flags, pcmk_action_runnable)) {

	pe_rsc_trace(instance, "Instance is starting due to %s",
	action->uuid);
	instance_state \|= instance_starting;
	} else {
	pe_rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}

	} else if (pcmk__str_eq(PCMK_ACTION_STOP, action->task,
	pcmk__str_none)) {
	/* Only stop actions can be pseudo-actions for primitives. That
	* indicates that the node they are on is being fenced, so the stop
	* is implied rather than actually executed.
	*/
	if (!optional
	&& pcmk_any_flags_set(action->flags, pcmk_action_pseudo
	\|pcmk_action_runnable)) {
	pe_rsc_trace(instance, "Instance is stopping due to %s",
	action->uuid);
	instance_state \|= instance_stopping;
	} else {
	pe_rsc_trace(instance, "%s doesn't affect %s state (%s)",
	action->uuid, instance->id,
	(optional? "optional" : "unrunnable"));
	}
	}
	}

	if (pcmk_all_flags_set(instance_state,
	instance_starting\|instance_stopping)) {
	instance_state \|= instance_restarting;
	}
	*state \|= instance_state;
	}

	/*!
	* \internal
	* \brief Create actions for collective resource instances
	*
	* \param[in,out] collective Clone or bundle resource to create actions for
	* \param[in,out] instances List of clone instances or bundle containers
	*/
	void
	pcmk__create_instance_actions(pe_resource_t collective, GList instances)
	{
	uint32_t state = 0;

	pe_action_t *stop = NULL;
	pe_action_t *stopped = NULL;

	pe_action_t *start = NULL;
	pe_action_t *started = NULL;

	pe_rsc_trace(collective, "Creating collective instance actions for %s",
	collective->id);

	// Create actions for each instance appropriate to its variant
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pe_resource_t instance = (pe_resource_t ) iter->data;

	instance->cmds->create_actions(instance);
	check_instance_state(instance, &state);
	}

	// Create pseudo-actions for rsc start and started
	start = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_START,
	!pcmk_is_set(state, instance_starting),
	true);
	started = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_RUNNING,
	!pcmk_is_set(state, instance_starting),
	false);
	started->priority = INFINITY;
	if (pcmk_any_flags_set(state, instance_active\|instance_starting)) {
	pe__set_action_flags(started, pcmk_action_runnable);
	}

	// Create pseudo-actions for rsc stop and stopped
	stop = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOP,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped = pe__new_rsc_pseudo_action(collective, PCMK_ACTION_STOPPED,
	!pcmk_is_set(state, instance_stopping),
	true);
	stopped->priority = INFINITY;
	if (!pcmk_is_set(state, instance_restarting)) {
	pe__set_action_flags(stop, pcmk_action_migratable);
	}

	if (collective->variant == pcmk_rsc_variant_clone) {
	pe__create_clone_notif_pseudo_ops(collective, start, started, stop,
	stopped);
	}
	}

	/*!
	* \internal
	* \brief Get a list of clone instances or bundle replica containers
	*
	* \param[in] rsc Clone or bundle resource
	*
	* \return Clone instances if \p rsc is a clone, or a newly created list of
	* \p rsc's replica containers if \p rsc is a bundle
	* \note The caller must call free_instance_list() on the result when the list
	* is no longer needed.
	*/
	static inline GList *
	get_instance_list(const pe_resource_t *rsc)
	{
	if (rsc->variant == pcmk_rsc_variant_bundle) {
	return pe__bundle_containers(rsc);
	} else {
	return rsc->children;
	}
	}

	/*!
	* \internal
	* \brief Free any memory created by get_instance_list()
	*
	* \param[in] rsc Clone or bundle resource passed to get_instance_list()
	* \param[in,out] list Return value of get_instance_list() for \p rsc
	*/
	static inline void
	free_instance_list(const pe_resource_t rsc, GList list)
	{
	if (list != rsc->children) {
	g_list_free(list);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance is compatible with a role and node
	*
	* \param[in] instance Clone instance or bundle replica container
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return true if \p instance is compatible with \p node and \p role,
	* otherwise false
	*/
	bool
	pcmk__instance_matches(const pe_resource_t instance, const pe_node_t node,
	enum rsc_role_e role, bool current)
	{
	pe_node_t *instance_node = NULL;

	CRM_CHECK((instance != NULL) && (node != NULL), return false);

	if ((role != pcmk_role_unknown)
	&& (role != instance->fns->state(instance, current))) {
	pe_rsc_trace(instance,
	"%s is not a compatible instance (role is not %s)",
	instance->id, role2text(role));
	return false;
	}

	if (!is_set_recursive(instance, pcmk_rsc_blocked, true)) {
	// We only want instances that haven't failed
	instance_node = instance->fns->location(instance, NULL, current);
	}

	if (instance_node == NULL) {
	pe_rsc_trace(instance,
	"%s is not a compatible instance (not assigned to a node)",
	instance->id);
	return false;
	}

	if (!pe__same_node(instance_node, node)) {
	pe_rsc_trace(instance,
	"%s is not a compatible instance (assigned to %s not %s)",
	instance->id, pe__node_name(instance_node),
	pe__node_name(node));
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Find an instance that matches a given resource by node and role
	*
	* \param[in] match_rsc Resource that instance must match (for logging only)
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] node Instance must match this node
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return \p rsc instance matching \p node and \p role if any, otherwise NULL
	*/
	static pe_resource_t *
	find_compatible_instance_on_node(const pe_resource_t *match_rsc,
	const pe_resource_t *rsc,
	const pe_node_t *node, enum rsc_role_e role,
	bool current)
	{
	GList *instances = NULL;

	instances = get_instance_list(rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pe_resource_t instance = (pe_resource_t ) iter->data;

	if (pcmk__instance_matches(instance, node, role, current)) {
	pe_rsc_trace(match_rsc,
	"Found %s %s instance %s compatible with %s on %s",
	role == pcmk_role_unknown? "matching" : role2text(role),
	rsc->id, instance->id, match_rsc->id,
	pe__node_name(node));
	free_instance_list(rsc, instances); // Only frees list, not contents
	return instance;
	}
	}
	free_instance_list(rsc, instances);

	pe_rsc_trace(match_rsc, "No %s %s instance found compatible with %s on %s",
	((role == pcmk_role_unknown)? "matching" : role2text(role)),
	rsc->id, match_rsc->id, pe__node_name(node));
	return NULL;
	}

	/*!
	* \internal
	* \brief Find a clone instance or bundle container compatible with a resource
	*
	* \param[in] match_rsc Resource that instance must match
	* \param[in] rsc Clone or bundle resource to check for matching instance
	* \param[in] role If not pcmk_role_unknown, instance must match this role
	* \param[in] current If true, compare instance's original node and role,
	* otherwise compare assigned next node and role
	*
	* \return Compatible (by \p role and \p match_rsc location) instance of \p rsc
	* if any, otherwise NULL
	*/
	pe_resource_t *
	pcmk__find_compatible_instance(const pe_resource_t *match_rsc,
	const pe_resource_t *rsc, enum rsc_role_e role,
	bool current)
	{
	pe_resource_t *instance = NULL;
	GList *nodes = NULL;
	const pe_node_t *node = match_rsc->fns->location(match_rsc, NULL, current);

	// If match_rsc has a node, check only that node
	if (node != NULL) {
	return find_compatible_instance_on_node(match_rsc, rsc, node, role,
	current);
	}

	// Otherwise check for an instance matching any of match_rsc's allowed nodes
	nodes = pcmk__sort_nodes(g_hash_table_get_values(match_rsc->allowed_nodes),
	NULL);
	for (GList *iter = nodes; (iter != NULL) && (instance == NULL);
	iter = iter->next) {
	instance = find_compatible_instance_on_node(match_rsc, rsc,
	(pe_node_t *) iter->data,
	role, current);
	}

	if (instance == NULL) {
	pe_rsc_debug(rsc, "No %s instance found compatible with %s",
	rsc->id, match_rsc->id);
	}
	g_list_free(nodes);
	return instance;
	}

	/*!
	* \internal
	* \brief Unassign an instance if mandatory ordering has no interleave match
	*
	* \param[in] first 'First' action in an ordering
	* \param[in] then 'Then' action in an ordering
	* \param[in,out] then_instance 'Then' instance that has no interleave match
	* \param[in] type Group of enum pcmk__action_relation_flags
	* \param[in] current If true, "then" action is stopped or demoted
	*
	* \return true if \p then_instance was unassigned, otherwise false
	*/
	static bool
	unassign_if_mandatory(const pe_action_t first, const pe_action_t then,
	pe_resource_t *then_instance, uint32_t type, bool current)
	{
	// Allow "then" instance to go down even without an interleave match
	if (current) {
	pe_rsc_trace(then->rsc,
	"%s has no instance to order before stopping "
	"or demoting %s",
	first->rsc->id, then_instance->id);

	/* If the "first" action must be runnable, but there is no "first"
	* instance, the "then" instance must not be allowed to come up.
	*/
	} else if (pcmk_any_flags_set(type, pe_order_runnable_left
	- \|pe_order_implies_then)) {
	+ \|pcmk__ar_first_implies_then)) {
	pe_rsc_info(then->rsc,
	"Inhibiting %s from being active "
	"because there is no %s instance to interleave",
	then_instance->id, first->rsc->id);
	return pcmk__assign_resource(then_instance, NULL, true, true);
	}
	return false;
	}

	/*!
	* \internal
	* \brief Find first matching action for a clone instance or bundle container
	*
	* \param[in] action Action in an interleaved ordering
	* \param[in] instance Clone instance or bundle container being interleaved
	* \param[in] action_name Action to look for
	* \param[in] node If not NULL, require action to be on this node
	* \param[in] for_first If true, \p instance is the 'first' resource in the
	* ordering, otherwise it is the 'then' resource
	*
	* \return First action for \p instance (or in some cases if \p instance is a
	* bundle container, its containerized resource) that matches
	* \p action_name and \p node if any, otherwise NULL
	*/
	static pe_action_t *
	find_instance_action(const pe_action_t action, const pe_resource_t instance,
	const char action_name, const pe_node_t node,
	bool for_first)
	{
	const pe_resource_t *rsc = NULL;
	pe_action_t *matching_action = NULL;

	/* If instance is a bundle container, sometimes we should interleave the
	* action for the container itself, and sometimes for the containerized
	* resource.
	*
	* For example, given "start bundle A then bundle B", B likely requires the
	* service inside A's container to be active, rather than just the
	* container, so we should interleave the action for A's containerized
	* resource. On the other hand, it's possible B's container itself requires
	* something from A, so we should interleave the action for B's container.
	*
	* Essentially, for 'first', we should use the containerized resource for
	* everything except stop, and for 'then', we should use the container for
	* everything except promote and demote (which can only be performed on the
	* containerized resource).
	*/
	if ((for_first && !pcmk__str_any_of(action->task, PCMK_ACTION_STOP,
	PCMK_ACTION_STOPPED, NULL))

	\|\| (!for_first && pcmk__str_any_of(action->task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_PROMOTED,
	PCMK_ACTION_DEMOTE,
	PCMK_ACTION_DEMOTED, NULL))) {

	rsc = pe__get_rsc_in_container(instance);
	}
	if (rsc == NULL) {
	rsc = instance; // No containerized resource, use instance itself
	} else {
	node = NULL; // Containerized actions are on bundle-created guest
	}

	matching_action = find_first_action(rsc->actions, NULL, action_name, node);
	if (matching_action != NULL) {
	return matching_action;
	}

	if (pcmk_is_set(instance->flags, pcmk_rsc_removed)
	\|\| pcmk__str_any_of(action_name, PCMK_ACTION_STOP, PCMK_ACTION_DEMOTE,
	NULL)) {
	crm_trace("No %s action found for %s%s",
	action_name,
	pcmk_is_set(instance->flags, pcmk_rsc_removed)? "orphan " : "",
	instance->id);
	} else {
	crm_err("No %s action found for %s to interleave (bug?)",
	action_name, instance->id);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get the original action name of a bundle or clone action
	*
	* Given an action for a bundle or clone, get the original action name,
	* mapping notify to the action being notified, and if the instances are
	* primitives, mapping completion actions to the action that was completed
	* (for example, stopped to stop).
	*
	* \param[in] action Clone or bundle action to check
	*
	* \return Original action name for \p action
	*/
	static const char *
	orig_action_name(const pe_action_t *action)
	{
	const pe_resource_t *instance = action->rsc->children->data; // Any instance
	char *action_type = NULL;
	const char *action_name = action->task;
	enum action_tasks orig_task = pcmk_action_unspecified;

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	// action->uuid is RSC_(confirmed-){pre,post}_notify_ACTION_INTERVAL
	CRM_CHECK(parse_op_key(action->uuid, NULL, &action_type, NULL),
	return task2text(pcmk_action_unspecified));
	action_name = strstr(action_type, "_notify_");
	CRM_CHECK(action_name != NULL,
	return task2text(pcmk_action_unspecified));
	action_name += strlen("_notify_");
	}
	orig_task = get_complex_task(instance, action_name);
	free(action_type);
	return task2text(orig_task);
	}

	/*!
	* \internal
	* \brief Update two interleaved actions according to an ordering between them
	*
	* Given information about an ordering of two interleaved actions, update the
	* actions' flags (and runnable_before members if appropriate) as appropriate
	* for the ordering. Effects may cascade to other orderings involving the
	* actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_interleaved_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t filter,
	uint32_t type)
	{
	GList *instances = NULL;
	uint32_t changed = pcmk__updated_none;
	const char *orig_first_task = orig_action_name(first);

	// Stops and demotes must be interleaved with instance on current node
	bool current = pcmk__ends_with(first->uuid, "_" PCMK_ACTION_STOPPED "_0")
	\|\| pcmk__ends_with(first->uuid,
	"_" PCMK_ACTION_DEMOTED "_0");

	// Update the specified actions for each "then" instance individually
	instances = get_instance_list(then->rsc);
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pe_resource_t *first_instance = NULL;
	pe_resource_t *then_instance = iter->data;

	pe_action_t *first_action = NULL;
	pe_action_t *then_action = NULL;

	// Find a "first" instance to interleave with this "then" instance
	first_instance = pcmk__find_compatible_instance(then_instance,
	first->rsc,
	pcmk_role_unknown,
	current);

	if (first_instance == NULL) { // No instance can be interleaved
	if (unassign_if_mandatory(first, then, then_instance, type,
	current)) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	continue;
	}

	first_action = find_instance_action(first, first_instance,
	orig_first_task, node, true);
	if (first_action == NULL) {
	continue;
	}

	then_action = find_instance_action(then, then_instance, then->task,
	node, false);
	if (then_action == NULL) {
	continue;
	}

	if (order_actions(first_action, then_action, type)) {
	pcmk__set_updated_flags(changed, first,
	pcmk__updated_first\|pcmk__updated_then);
	}

	changed \|= then_instance->cmds->update_ordered_actions(
	first_action, then_action, node,
	first_instance->cmds->action_flags(first_action, node), filter,
	type, then->rsc->cluster);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}

	/*!
	* \internal
	* \brief Check whether two actions in an ordering can be interleaved
	*
	* \param[in] first 'First' action in the ordering
	* \param[in] then 'Then' action in the ordering
	*
	* \return true if \p first and \p then can be interleaved, otherwise false
	*/
	static bool
	can_interleave_actions(const pe_action_t first, const pe_action_t then)
	{
	bool interleave = false;
	pe_resource_t *rsc = NULL;

	if ((first->rsc == NULL) \|\| (then->rsc == NULL)) {
	crm_trace("Not interleaving %s with %s: not resource actions",
	first->uuid, then->uuid);
	return false;
	}

	if (first->rsc == then->rsc) {
	crm_trace("Not interleaving %s with %s: same resource",
	first->uuid, then->uuid);
	return false;
	}

	if ((first->rsc->variant < pcmk_rsc_variant_clone)
	\|\| (then->rsc->variant < pcmk_rsc_variant_clone)) {
	crm_trace("Not interleaving %s with %s: not clones or bundles",
	first->uuid, then->uuid);
	return false;
	}

	if (pcmk__ends_with(then->uuid, "_stop_0")
	\|\| pcmk__ends_with(then->uuid, "_demote_0")) {
	rsc = first->rsc;
	} else {
	rsc = then->rsc;
	}

	interleave = crm_is_true(g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INTERLEAVE));
	pe_rsc_trace(rsc, "'%s then %s' will %sbe interleaved (based on %s)",
	first->uuid, then->uuid, (interleave? "" : "not "), rsc->id);
	return interleave;
	}

	/*!
	* \internal
	* \brief Update non-interleaved instance actions according to an ordering
	*
	* Given information about an ordering of two non-interleaved actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] instance Clone instance or bundle container
	* \param[in,out] first "First" action in ordering
	* \param[in] then "Then" action in ordering (for \p instance's parent)
	* \param[in] node If not NULL, limit scope of ordering to this node
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static uint32_t
	update_noninterleaved_actions(pe_resource_t instance, pe_action_t first,
	const pe_action_t then, const pe_node_t node,
	uint32_t flags, uint32_t filter, uint32_t type)
	{
	pe_action_t *instance_action = NULL;
	uint32_t instance_flags = 0;
	uint32_t changed = pcmk__updated_none;

	// Check whether instance has an equivalent of "then" action
	instance_action = find_first_action(instance->actions, NULL, then->task,
	node);
	if (instance_action == NULL) {
	return changed;
	}

	// Check whether action is runnable
	instance_flags = instance->cmds->action_flags(instance_action, node);
	if (!pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	return changed;
	}

	// If so, update actions for the instance
	changed = instance->cmds->update_ordered_actions(first, instance_action,
	node, flags, filter, type,
	instance->cluster);

	// Propagate any changes to later actions
	if (pcmk_is_set(changed, pcmk__updated_then)) {
	for (GList *after_iter = instance_action->actions_after;
	after_iter != NULL; after_iter = after_iter->next) {
	pe_action_wrapper_t *after = after_iter->data;

	pcmk__update_action_for_orderings(after->action, instance->cluster);
	}
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two clone or bundle actions, update
	* the actions' flags (and runnable_before members if appropriate) as
	* appropriate for the ordering. Effects may cascade to other orderings
	* involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__instance_update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	CRM_ASSERT((first != NULL) && (then != NULL) && (data_set != NULL));

	if (then->rsc == NULL) {
	return pcmk__updated_none;

	} else if (can_interleave_actions(first, then)) {
	return update_interleaved_actions(first, then, node, filter, type);

	} else {
	uint32_t changed = pcmk__updated_none;
	GList *instances = get_instance_list(then->rsc);

	// Update actions for the clone or bundle resource itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags,
	filter, type, data_set);

	// Update the 'then' clone instances or bundle containers individually
	for (GList *iter = instances; iter != NULL; iter = iter->next) {
	pe_resource_t *instance = iter->data;

	changed \|= update_noninterleaved_actions(instance, first, then,
	node, flags, filter, type);
	}
	free_instance_list(then->rsc, instances);
	return changed;
	}
	}

	#define pe__clear_action_summary_flags(flags, action, flag) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Action summary", action->rsc->id, \
	flags, flag, #flag); \
	} while (0)

	/*!
	* \internal
	* \brief Return action flags for a given clone or bundle action
	*
	* \param[in,out] action Action for a clone or bundle
	* \param[in] instances Clone instances or bundle containers
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__collective_action_flags(pe_action_t action, const GList instances,
	const pe_node_t *node)
	{
	bool any_runnable = false;
	const char *action_name = orig_action_name(action);

	// Set original assumptions (optional and runnable may be cleared below)
	uint32_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	for (const GList *iter = instances; iter != NULL; iter = iter->next) {
	const pe_resource_t *instance = iter->data;
	const pe_node_t *instance_node = NULL;
	pe_action_t *instance_action = NULL;
	uint32_t instance_flags;

	// Node is relevant only to primitive instances
	if (instance->variant == pcmk_rsc_variant_primitive) {
	instance_node = node;
	}

	instance_action = find_first_action(instance->actions, NULL,
	action_name, instance_node);
	if (instance_action == NULL) {
	pe_rsc_trace(action->rsc, "%s has no %s action on %s",
	instance->id, action_name, pe__node_name(node));
	continue;
	}

	pe_rsc_trace(action->rsc, "%s has %s for %s on %s",
	instance->id, instance_action->uuid, action_name,
	pe__node_name(node));

	instance_flags = instance->cmds->action_flags(instance_action, node);

	// If any instance action is mandatory, so is the collective action
	if (pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(instance_flags, pcmk_action_optional)) {
	pe_rsc_trace(instance, "%s is mandatory because %s is",
	action->uuid, instance_action->uuid);
	pe__clear_action_summary_flags(flags, action,
	pcmk_action_optional);
	pe__clear_action_flags(action, pcmk_action_optional);
	}

	// If any instance action is runnable, so is the collective action
	if (pcmk_is_set(instance_flags, pcmk_action_runnable)) {
	any_runnable = true;
	}
	}

	if (!any_runnable) {
	pe_rsc_trace(action->rsc,
	"%s is not runnable because no instance can run %s",
	action->uuid, action_name);
	pe__clear_action_summary_flags(flags, action, pcmk_action_runnable);
	if (node == NULL) {
	pe__clear_action_flags(action, pcmk_action_runnable);
	}
	}

	return flags;
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index c6fc652030..6a9ad13ba2 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1488 +1,1490 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(data_set->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	crm_warn("Unknown action '%s' specified in order constraint", action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pe_wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pcmk__ar_none; // so we trace-log all flags set

	pe__set_order_flags(flags, pcmk__ar_ordered);

	switch (kind) {
	case pe_order_kind_optional:
	break;

	case pe_order_kind_serialize:
	pe__set_order_flags(flags, pe_order_serialize_only);
	break;

	case pe_order_kind_mandatory:
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pe_order_asymmetrical);
	break;

	case ordering_symmetric:
	- pe__set_order_flags(flags, pe_order_implies_then);
	+ pe__set_order_flags(flags, pcmk__ar_first_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pe__set_order_flags(flags, pe_order_runnable_left);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pcmk__ar_then_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] data_set Cluster working set
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pe_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pe_working_set_t *data_set)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(data_set->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pe_warn_once(pe_wo_order_inst,
	"Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
	XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pe_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta, PCMK_META_CLONE_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pe_warn_once(pe_wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pe_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pcmk_action_min_runnable);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
	pe_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	rsc_first->cluster);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pe_order_runnable_left, rsc_first->cluster);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pe_working_set_t data_set)
	{
	pe_resource_t *rsc_then = NULL;
	pe_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pcmk__ar_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	data_set);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	data_set);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	- handle_restart_type(rsc_then, kind, pe_order_implies_then, flags);
	+ handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Group of enum pcmk__action_relation_flags
	* \param[in,out] sched Cluster working set to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pe_resource_t first_rsc, char first_action_task,
	pe_action_t first_action, pe_resource_t then_rsc,
	char then_action_task, pe_action_t then_action,
	uint32_t flags, pe_working_set_t *sched)
	{
	pe__ordering_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pe__ordering_t));
	CRM_ASSERT(order != NULL);

	order->id = sched->order_id++;
	order->flags = flags;
	order->lh_rsc = first_rsc;
	order->rh_rsc = then_rsc;
	order->lh_action = first_action;
	order->rh_action = then_action;
	order->lh_action_task = first_action_task;
	order->rh_action_task = then_action_task;

	if ((order->lh_action_task == NULL) && (first_action != NULL)) {
	order->lh_action_task = strdup(first_action->uuid);
	}

	if ((order->rh_action_task == NULL) && (then_action != NULL)) {
	order->rh_action_task = strdup(then_action->uuid);
	}

	if ((order->lh_rsc == NULL) && (first_action != NULL)) {
	order->lh_rsc = first_action->rsc;
	}

	if ((order->rh_rsc == NULL) && (then_action != NULL)) {
	order->rh_rsc = then_action->rsc;
	}

	pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->order_id - 1),
	pcmk__s(order->lh_action_task, "an underspecified action"),
	pcmk__s(order->rh_action_task, "an underspecified action"));

	sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pe_working_set_t data_set)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pe_resource_t *last = NULL;
	pe_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pcmk__ar_ordered;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pe_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, data_set);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] data_set Cluster working set
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pe_working_set_t *data_set,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pe_resource_t *rsc_1 = NULL;
	pe_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pcmk__ar_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
	pe_action_t *unordered_action = get_pseudo_op(task, data_set);

	free(task);
	pe__set_action_flags(unordered_action, pcmk_action_min_runnable);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pe_order_one_or_more
	\|pe_order_implies_then_printed,
	data_set);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2,
	XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pe_working_set_t *data_set)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pe_resource_t *rsc_first = NULL;
	pe_resource_t *rsc_then = NULL;
	pe_tag_t *tag_first = NULL;
	pe_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, data_set) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, data_set,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, data_set,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, data_set);
	}
	}

	static bool
	ordering_is_invalid(pe_action_t action, pe_action_wrapper_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pe_order_preserve)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->type == pe_order_load) && action->rsc
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = pcmk__ar_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pe_node_t node, pe_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pe_action_t action = (pe_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if (!pe__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk_rsc_managed\|pcmk_rsc_blocked)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pe__node_name(node));
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pcmk_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pcmk__ar_ordered\|pe_order_runnable_left,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pe_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->actions, key, NULL);
	free(key);
	free(task);
	} else {
	crm_err("Invalid operation key (bug?): %s", original_key);
	}
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pe_action_t *first_action,
	const pe_resource_t rsc, pe__ordering_t order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pcmk__ar_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->rh_action != NULL) {
	then_actions = g_list_prepend(NULL, order->rh_action);

	} else {
	then_actions = find_actions_by_task(rsc, order->rh_action_task);
	}

	if (then_actions == NULL) {
	pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->rh_action_task, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pcmk_action_migration_abort)) {

	pe_rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->rh_action_task, rsc->id);
	- pe__clear_order_flags(flags, pe_order_implies_then);
	+ pe__clear_order_flags(flags, pcmk__ar_first_implies_then);
	}

	- if ((first_action == NULL) && !pcmk_is_set(flags, pe_order_implies_then)) {
	+ if ((first_action == NULL)
	+ && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {
	+
	pe_rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pe_action_t then_action_iter = (pe_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pe__clear_action_flags(then_action_iter, pcmk_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->lh_action_task, order->lh_rsc->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pe_resource_t first_rsc, pe__ordering_t order)
	{
	GList *first_actions = NULL;
	pe_action_t *first_action = order->lh_action;
	pe_resource_t *then_rsc = order->rh_rsc;

	CRM_ASSERT(first_rsc != NULL);
	pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->lh_action_task);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc,
	TRUE) == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else {
	pe_rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->lh_action_task, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	TRUE, first_rsc->cluster);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->rh_action == NULL) {
	pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->rh_action->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->rh_action, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pe_action_t *) data,
	(pe_working_set_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Cluster working set
	*/
	void
	pcmk__apply_orderings(pe_working_set_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);

	for (GList *iter = sched->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pe__ordering_t *order = iter->data;
	pe_resource_t *rsc = order->lh_rsc;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	order_resource_actions_after(order->lh_action, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->lh_action, order->rh_action, order->flags);
	}
	}

	g_list_foreach(sched->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->actions));
	g_list_foreach(sched->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pe_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pe_action_t before = (pe_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pe__node_name(before->node),
	after_desc, pe__node_name(after->node));
	order_actions(before, after, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pe_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_ordered);
	}
	diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
	index a0712b84d6..c8eecd03c3 100644
	--- a/lib/pacemaker/pcmk_sched_primitive.c
	+++ b/lib/pacemaker/pcmk_sched_primitive.c
	@@ -1,1652 +1,1655 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdint.h> // uint8_t, uint32_t

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static void stop_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void start_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void demote_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void promote_resource(pe_resource_t rsc, pe_node_t node,
	bool optional);
	static void assert_role_error(pe_resource_t rsc, pe_node_t node,
	bool optional);

	#define RSC_ROLE_MAX (pcmk_role_promoted + 1)

	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the immediate next role when transitioning from one role
	* to a target role. For example, when going from Stopped to Promoted, the
	* next role is Unpromoted, because the resource must be started before it
	* can be promoted. The current state then becomes Started, which is fed
	* into this array again, giving a next role of Promoted.
	*
	* Current role Immediate next role Final target role
	* ------------ ------------------- -----------------
	*/
	/* Unknown / { pcmk_role_unknown, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_stopped, /* Unpromoted */
	pcmk_role_stopped, /* Promoted */
	},
	/* Stopped / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_unpromoted, /* Promoted */
	},
	/* Started / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Unpromoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Promoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_unpromoted, /* Stopped */
	pcmk_role_unpromoted, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Function to schedule actions needed for a role change
	*
	* \param[in,out] rsc Resource whose role is changing
	* \param[in,out] node Node where resource will be in its next role
	* \param[in] optional Whether scheduled actions should be optional
	*/
	typedef void (rsc_transition_fn)(pe_resource_t rsc, pe_node_t *node,
	bool optional);

	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the function needed to transition directly from one role
	* to another. NULL indicates that nothing is needed.
	*
	* Current role Transition function Next role
	* ------------ ------------------- ----------
	*/
	/* Unknown / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	assert_role_error, /* Started */
	assert_role_error, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Stopped / { assert_role_error, / Unknown */
	NULL, /* Stopped */
	start_resource, /* Started */
	start_resource, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Started / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	NULL, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Unpromoted / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	stop_resource, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Promoted / { assert_role_error, / Unknown */
	demote_resource, /* Stopped */
	demote_resource, /* Started */
	demote_resource, /* Unpromoted */
	NULL, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Get a list of a resource's allowed nodes sorted by node score
	*
	* \param[in] rsc Resource to check
	*
	* \return List of allowed nodes sorted by node score
	*/
	static GList *
	sorted_allowed_nodes(const pe_resource_t *rsc)
	{
	if (rsc->allowed_nodes != NULL) {
	GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);

	if (nodes != NULL) {
	return pcmk__sort_nodes(nodes, pe__current_node(rsc));
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Assign a resource to its best allowed node, if possible
	*
	* \param[in,out] rsc Resource to choose a node for
	* \param[in] prefer If not \c NULL, prefer this node when all else
	* equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return true if \p rsc could be assigned to a node, otherwise false
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	static bool
	assign_best_node(pe_resource_t rsc, const pe_node_t prefer, bool stop_if_fail)
	{
	GList *nodes = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *best = NULL;
	const pe_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);

	if (prefer == NULL) {
	prefer = most_free_node;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// We've already finished assignment of resources to nodes
	return rsc->allocated_to != NULL;
	}

	// Sort allowed nodes by score
	nodes = sorted_allowed_nodes(rsc);
	if (nodes != NULL) {
	best = (pe_node_t *) nodes->data; // First node has best score
	}

	if ((prefer != NULL) && (nodes != NULL)) {
	// Get the allowed node version of prefer
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
	pe__node_name(prefer), rsc->id);

	/* Favor the preferred node as long as its score is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's score is less than INFINITY.
	*/
	} else if (chosen->weight < best->weight) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen, true, false)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else {
	pe_rsc_trace(rsc,
	"Chose preferred node %s for %s "
	"(ignoring %d candidates)",
	pe__node_name(chosen), rsc->id, g_list_length(nodes));
	}
	}

	if ((chosen == NULL) && (best != NULL)) {
	/* Either there is no preferred node, or the preferred node is not
	* suitable, but another node is allowed to run the resource.
	*/

	chosen = best;

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& (chosen->weight > 0) // Zero not acceptable
	&& pcmk__node_available(chosen, false, false)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* pcmk__assign_instances() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unassigned instances to prefer a node that's already
	* running another instance.
	*/
	pe_node_t *running = pe__current_node(rsc);

	if (running == NULL) {
	// Nothing to do

	} else if (!pcmk__node_available(running, true, false)) {
	pe_rsc_trace(rsc,
	"Current node for %s (%s) can't run resources",
	rsc->id, pe__node_name(running));

	} else {
	int nodes_with_best_score = 1;

	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pe_node_t allowed = (pe_node_t ) iter->data;

	if (allowed->weight != chosen->weight) {
	// The nodes are sorted by score, so no more are equal
	break;
	}
	if (pe__same_node(allowed, running)) {
	// Scores are equal, so prefer the current node
	chosen = allowed;
	}
	nodes_with_best_score++;
	}

	if (nodes_with_best_score > 1) {
	uint8_t log_level = LOG_INFO;

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}
	do_crm_log(log_level,
	"Chose %s for %s from %d nodes with score %s",
	pe__node_name(chosen), rsc->id,
	nodes_with_best_score,
	pcmk_readable_score(chosen->weight));
	}
	}
	}

	pe_rsc_trace(rsc, "Chose %s for %s from %d candidates",
	pe__node_name(chosen), rsc->id, g_list_length(nodes));
	}

	pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
	g_list_free(nodes);
	return rsc->allocated_to != NULL;
	}

	/*!
	* \internal
	* \brief Apply a "this with" colocation to a node's allowed node scores
	*
	* \param[in,out] colocation Colocation to apply
	* \param[in,out] rsc Resource being assigned
	*/
	static void
	apply_this_with(pcmk__colocation_t colocation, pe_resource_t rsc)
	{
	GHashTable *archive = NULL;
	pe_resource_t *other = colocation->primary;

	// In certain cases, we will need to revert the node scores
	if ((colocation->dependent_role >= pcmk_role_promoted)
	\|\| ((colocation->score < 0) && (colocation->score > -INFINITY))) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	if (pcmk_is_set(other->flags, pcmk_rsc_unassigned)) {
	pe_rsc_trace(rsc,
	"%s: Assigning colocation %s primary %s first"
	"(score=%d role=%s)",
	rsc->id, colocation->id, other->id,
	colocation->score, role2text(colocation->dependent_role));
	other->cmds->assign(other, NULL, true);
	}

	// Apply the colocation score to this resource's allowed node scores
	rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
	if ((archive != NULL)
	&& !pcmk__any_node_available(rsc->allowed_nodes)) {
	pe_rsc_info(rsc,
	"%s: Reverting scores from colocation with %s "
	"because no nodes allowed",
	rsc->id, other->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive != NULL) {
	g_hash_table_destroy(archive);
	}
	}

	/*!
	* \internal
	* \brief Update a Pacemaker Remote node once its connection has been assigned
	*
	* \param[in] connection Connection resource that has been assigned
	*/
	static void
	remote_connection_assigned(const pe_resource_t *connection)
	{
	pe_node_t *remote_node = pe_find_node(connection->cluster->nodes,
	connection->id);

	CRM_CHECK(remote_node != NULL, return);

	if ((connection->allocated_to != NULL)
	&& (connection->next_role != pcmk_role_stopped)) {

	crm_trace("Pacemaker Remote node %s will be online",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	if (remote_node->details->unseen) {
	// Avoid unnecessary fence, since we will attempt connection
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Pacemaker Remote node %s will be shut down "
	"(%sassigned connection's next role is %s)",
	remote_node->details->id,
	((connection->allocated_to == NULL)? "un" : ""),
	role2text(connection->next_role));
	remote_node->details->shutdown = TRUE;
	}
	}

	/*!
	* \internal
	* \brief Assign a primitive resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pe_node_t *
	pcmk__primitive_assign(pe_resource_t rsc, const pe_node_t prefer,
	bool stop_if_fail)
	{
	GList *this_with_colocations = NULL;
	GList *with_this_colocations = NULL;
	GList *iter = NULL;
	pcmk__colocation_t *colocation = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	// Never assign a child without parent being assigned first
	if ((rsc->parent != NULL)
	&& !pcmk_is_set(rsc->parent->flags, pcmk_rsc_assigning)) {
	pe_rsc_debug(rsc, "%s: Assigning parent %s first",
	rsc->id, rsc->parent->id);
	rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// Assignment has already been done
	const char *node_name = "no node";

	if (rsc->allocated_to != NULL) {
	node_name = pe__node_name(rsc->allocated_to);
	}
	pe_rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
	return rsc->allocated_to;
	}

	// Ensure we detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pe_rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pe__set_resource_flags(rsc, pcmk_rsc_assigning);

	pe__show_node_scores(true, rsc, "Pre-assignment", rsc->allowed_nodes,
	rsc->cluster);

	this_with_colocations = pcmk__this_with_colocations(rsc);
	with_this_colocations = pcmk__with_this_colocations(rsc);

	// Apply mandatory colocations first, to satisfy as many as possible
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	pe__show_node_scores(true, rsc, "Mandatory-colocations",
	rsc->allowed_nodes, rsc->cluster);

	// Then apply optional colocations
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	g_list_free(this_with_colocations);
	g_list_free(with_this_colocations);

	if (rsc->next_role == pcmk_role_stopped) {
	pe_rsc_trace(rsc,
	"Banning %s from all nodes because it will be stopped",
	rsc->id);
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE,
	rsc->cluster);

	} else if ((rsc->next_role > rsc->role)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_quorate)
	&& (rsc->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Resource %s cannot be elevated from %s to %s due to "
	"no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
	}

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	// Unmanage resource if fencing is enabled but no device is configured
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_have_fencing)) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	// Unmanaged resources stay on their current node
	const char *reason = NULL;
	pe_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == pcmk_role_promoted) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"), reason);
	pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);

	} else if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_stop_all)) {
	// Must stop at some point, but be consistent with stop_if_fail
	if (stop_if_fail) {
	pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources",
	rsc->id);
	}
	pcmk__assign_resource(rsc, NULL, true, stop_if_fail);

	} else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
	// Assignment failed
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if ((rsc->running_on != NULL) && stop_if_fail) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}
	}

	pe__clear_resource_flags(rsc, pcmk_rsc_assigning);

	if (rsc->is_remote_node) {
	remote_connection_assigned(rsc);
	}

	return rsc->allocated_to;
	}

	/*!
	* \internal
	* \brief Schedule actions to bring resource down and back to current role
	*
	* \param[in,out] rsc Resource to restart
	* \param[in,out] current Node that resource should be brought down on
	* \param[in] need_stop Whether the resource must be stopped
	* \param[in] need_promote Whether the resource must be promoted
	*
	* \return Role that resource would have after scheduled actions are taken
	*/
	static void
	schedule_restart_actions(pe_resource_t rsc, pe_node_t current,
	bool need_stop, bool need_promote)
	{
	enum rsc_role_e role = rsc->role;
	enum rsc_role_e next_role;
	rsc_transition_fn fn = NULL;

	pe__set_resource_flags(rsc, pcmk_rsc_restarting);

	// Bring resource down to a stop on its current node
	while (role != pcmk_role_stopped) {
	next_role = rsc_state_matrix[role][pcmk_role_stopped];
	pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, current, !need_stop);
	role = next_role;
	}

	// Bring resource up to its next role on its next node
	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == pcmk_role_promoted) && need_promote) {
	required = true;
	}
	pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, !required);
	role = next_role;
	}

	pe__clear_resource_flags(rsc, pcmk_rsc_restarting);
	}

	/*!
	* \internal
	* \brief If a resource's next role is not explicitly specified, set a default
	*
	* \param[in,out] rsc Resource to set next role for
	*
	* \return "explicit" if next role was explicitly set, otherwise "implicit"
	*/
	static const char *
	set_default_next_role(pe_resource_t *rsc)
	{
	if (rsc->next_role != pcmk_role_unknown) {
	return "explicit";
	}

	if (rsc->allocated_to == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
	} else {
	pe__set_next_role(rsc, pcmk_role_started, "assignment");
	}
	return "implicit";
	}

	/*!
	* \internal
	* \brief Create an action to represent an already pending start
	*
	* \param[in,out] rsc Resource to create start action for
	*/
	static void
	create_pending_start(pe_resource_t *rsc)
	{
	pe_action_t *start = NULL;

	pe_rsc_trace(rsc,
	"Creating action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	pe__set_action_flags(start, pcmk_action_always_in_graph);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to take a resource to its next role
	*
	* \param[in,out] rsc Resource to schedule actions for
	*/
	static void
	schedule_role_transition_actions(pe_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	while (role != rsc->next_role) {
	enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
	rsc_transition_fn fn = NULL;

	pe_rsc_trace(rsc,
	"Creating action to take %s from %s to %s (ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, false);
	role = next_role;
	}
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given primitive resource
	*
	* \param[in,out] rsc Primitive resource to create actions for
	*/
	void
	pcmk__primitive_create_actions(pe_resource_t *rsc)
	{
	bool need_stop = false;
	bool need_promote = false;
	bool is_moving = false;
	bool allow_migrate = false;
	bool multiply_active = false;

	pe_node_t *current = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	const char *next_role_source = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	next_role_source = set_default_next_role(rsc);
	pe_rsc_trace(rsc,
	"Creating all actions for %s transition from %s to %s "
	"(%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	next_role_source, pe__node_name(rsc->allocated_to));

	current = rsc->fns->active_node(rsc, &num_all_active, &num_clean_active);

	g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
	rsc);

	if ((current != NULL) && (rsc->allocated_to != NULL)
	&& !pe__same_node(current, rsc->allocated_to)
	&& (rsc->next_role >= pcmk_role_started)) {

	pe_rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, pe__node_name(current),
	pe__node_name(rsc->allocated_to));
	is_moving = true;
	allow_migrate = pcmk__rsc_can_migrate(rsc, current);

	// This is needed even if migrating (though I'm not sure why ...)
	need_stop = true;
	}

	// Check whether resource is partially migrated and/or multiply active
	if ((rsc->partial_migration_source != NULL)
	&& (rsc->partial_migration_target != NULL)
	&& allow_migrate && (num_all_active == 2)
	&& pe__same_node(current, rsc->partial_migration_source)
	&& pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
	/* A partial migration is in progress, and the migration target remains
	* the same as when the migration began.
	*/
	pe_rsc_trace(rsc, "Partial migration of %s from %s to %s will continue",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));

	} else if ((rsc->partial_migration_source != NULL)
	\|\| (rsc->partial_migration_target != NULL)) {
	// A partial migration is in progress but can't be continued

	if (num_all_active > 2) {
	// The resource is migrating and multiply active!
	crm_notice("Forcing recovery of %s because it is migrating "
	"from %s to %s and possibly active elsewhere",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	} else {
	// The migration source or target isn't available
	crm_notice("Forcing recovery of %s because it can no longer "
	"migrate from %s to %s",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	}
	need_stop = true;
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = false;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	multiply_active = (num_all_active > 1);
	} else {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* stonith-enabled is false). We can start such resources elsewhere
	* before fencing completes, and if we considered the resource active on
	* the failed node, we would attempt recovery for being active on
	* multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	}

	if (multiply_active) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pe_proc_err("%s resource %s might be active on %u nodes (%s)",
	pcmk__s(class, "Untyped"), rsc->id, num_all_active,
	recovery2text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
	"#Resource_is_Too_Active for more information");

	switch (rsc->recovery_type) {
	case pcmk_multiply_active_restart:
	need_stop = true;
	break;
	case pcmk_multiply_active_unexpected:
	need_stop = true; // stop_resource() will skip expected node
	pe__set_resource_flags(rsc, pcmk_rsc_stop_unexpected);
	break;
	default:
	break;
	}

	} else {
	pe__clear_resource_flags(rsc, pcmk_rsc_stop_unexpected);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	create_pending_start(rsc);
	}

	if (is_moving) {
	// Remaining tests are only for resources staying where they are

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_stop_if_failed)) {
	need_stop = true;
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == pcmk_role_promoted) {
	need_promote = true;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = true;

	} else if ((rsc->role > pcmk_role_started) && (current != NULL)
	&& (rsc->allocated_to != NULL)) {
	pe_action_t *start = NULL;

	pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	if (!pcmk_is_set(start->flags, pcmk_action_optional)) {
	// Recovery of a promoted resource
	pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = true;
	}
	}

	// Create any actions needed to bring resource down and back up to same role
	schedule_restart_actions(rsc, current, need_stop, need_promote);

	// Create any actions needed to take resource from this role to the next
	schedule_role_transition_actions(rsc);

	pcmk__create_recurring_actions(rsc);

	if (allow_migrate) {
	pcmk__create_migration_actions(rsc, current);
	}
	}

	/*!
	* \internal
	* \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
	*
	* \param[in] rsc Resource to check
	*/
	static void
	rsc_avoids_remote_nodes(const pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (node->details->remote_rsc != NULL) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(const pe_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
	}

	return allowed_nodes;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a primitive resource
	*
	* \param[in,out] rsc Primitive resource to create implicit constraints for
	*/
	void
	pcmk__primitive_internal_constraints(pe_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;
	bool check_unfencing = false;
	bool check_utilization = false;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping implicit constraints for unmanaged resource %s",
	rsc->id);
	return;
	}

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)
	&& pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(rsc->cluster->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
	- pcmk__ar_ordered\|pe_order_implies_then\|pe_order_restart,
	+ pcmk__ar_ordered
	+ \|pcmk__ar_first_implies_then
	+ \|pe_order_restart,
	rsc->cluster);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)
	\|\| (rsc->role > pcmk_role_unpromoted)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	pe_order_promoted_implies_first, rsc->cluster);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
	NULL,
	pe_order_runnable_left, rsc->cluster);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL, pe_order_same_node\|pe_order_then_cancels_first,
	rsc->cluster);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| (rsc->container != NULL)) {
	allowed_nodes = allowed_nodes_as_list(rsc);
	}

	if (check_unfencing) {
	g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->container != NULL) {
	pe_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Guest resources are not allowed to run on Pacemaker Remote nodes,
	* to avoid nesting remotes. However, bundles are allowed.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->container, PCMK_ACTION_MONITOR,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
	rsc->container);
	}

	if (remote_rsc != NULL) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	pcmk__new_ordering(rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_START, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	- pe_order_implies_then\|pe_order_runnable_left,
	+ pcmk__ar_first_implies_then
	+ \|pe_order_runnable_left,
	rsc->cluster);

	pcmk__new_ordering(rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_STOP, 0),
	NULL, pcmk__ar_then_implies_first, rsc->cluster);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL,
	pcmk__coloc_influence);
	}
	}

	if (rsc->is_remote_node
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	/* Remote connections and fencing devices are not allowed to run on
	* Pacemaker Remote nodes
	*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__primitive_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	// Always process on behalf of primary resource
	primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
	return;
	}

	filter_results = pcmk__colocation_affects(dependent, primary, colocation,
	false);
	pe_rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((colocation->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, colocation->id, colocation->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, colocation);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_scores(dependent, primary, colocation);
	break;
	default: // pcmk__coloc_affects_nothing
	return;
	}
	}

	/* Primitive implementation of
	* resource_alloc_functions_t:with_this_colocations()
	*/
	void
	pcmk__with_primitive_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_with_this(list, colocation, orig_rsc);
	}
	}
	}
	}

	/* Primitive implementation of
	* resource_alloc_functions_t:this_with_colocations()
	*/
	void
	pcmk__primitive_with_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given primitive resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node (ignored)
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__primitive_action_flags(pe_action_t action, const pe_node_t node)
	{
	CRM_ASSERT(action != NULL);
	return (uint32_t) action->flags;
	}

	/*!
	* \internal
	* \brief Check whether a node is a multiply active resource's expected node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p rsc is multiply active with multiple-active set to
	* stop_unexpected, and \p node is the node where it will remain active
	* \note This assumes that the resource's next role cannot be changed to stopped
	* after this is called, which should be reasonable if status has already
	* been unpacked and resources have been assigned to nodes.
	*/
	static bool
	is_expected_node(const pe_resource_t rsc, const pe_node_t node)
	{
	return pcmk_all_flags_set(rsc->flags,
	pcmk_rsc_stop_unexpected\|pcmk_rsc_restarting)
	&& (rsc->next_role > pcmk_role_stopped)
	&& pe__same_node(rsc->allocated_to, node);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to stop a resource wherever it is active
	*
	* \param[in,out] rsc Resource being stopped
	* \param[in] node Node where resource is being stopped (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	stop_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t current = (pe_node_t ) iter->data;
	pe_action_t *stop = NULL;

	if (is_expected_node(rsc, current)) {
	/* We are scheduling restart actions for a multiply active resource
	* with multiple-active=stop_unexpected, and this is where it should
	* not be stopped.
	*/
	pe_rsc_trace(rsc,
	"Skipping stop of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	continue;
	}

	if (rsc->partial_migration_target != NULL) {
	// Continue migration if node originally was and remains target
	if (pe__same_node(current, rsc->partial_migration_target)
	&& pe__same_node(current, rsc->allocated_to)) {
	pe_rsc_trace(rsc,
	"Skipping stop of %s on %s "
	"because partial migration there will continue",
	rsc->id, pe__node_name(current));
	continue;
	} else {
	pe_rsc_trace(rsc,
	"Forcing stop of %s on %s "
	"because migration target changed",
	rsc->id, pe__node_name(current));
	optional = false;
	}
	}

	pe_rsc_trace(rsc, "Scheduling stop of %s on %s",
	rsc->id, pe__node_name(current));
	stop = stop_action(rsc, current, optional);

	if (rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", true);
	} else if (pcmk_all_flags_set(rsc->flags, pcmk_rsc_restarting
	\|pcmk_rsc_stop_unexpected)) {
	/* We are stopping a multiply active resource on a node that is
	* not its expected node, and we are still scheduling restart
	* actions, so the stop is for being multiply active.
	*/
	pe_action_set_reason(stop, "being multiply active", true);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe__clear_action_flags(stop, pcmk_action_runnable);
	}

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_remove_after_stop)) {
	pcmk__schedule_cleanup(rsc, current, optional);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing)) {
	pe_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
	NULL, false, rsc->cluster);

	order_actions(stop, unfence, pcmk__ar_then_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pe_proc_err("Stopping %s until %s can be unfenced",
	rsc->id, pe__node_name(current));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to start a resource on a node
	*
	* \param[in,out] rsc Resource being started
	* \param[in,out] node Node where resource should be started
	* \param[in] optional Whether actions should be optional
	*/
	static void
	start_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	pe_action_t *start = NULL;

	CRM_ASSERT(node != NULL);

	pe_rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node), node->weight);
	start = start_action(rsc, node, TRUE);

	- pcmk__order_vs_unfence(rsc, node, start, pe_order_implies_then);
	+ pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);

	if (pcmk_is_set(start->flags, pcmk_action_runnable) && !optional) {
	pe__clear_action_flags(start, pcmk_action_optional);
	}

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the start becomes necessary for other
	* reasons later.
	*/
	pe_rsc_trace(rsc,
	"Start of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pe__set_action_flags(start, pcmk_action_pseudo);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to promote a resource on a node
	*
	* \param[in,out] rsc Resource being promoted
	* \param[in] node Node where resource should be promoted
	* \param[in] optional Whether actions should be optional
	*/
	static void
	promote_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool runnable = true;

	CRM_ASSERT(node != NULL);

	// Any start must be runnable for promotion to be runnable
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t start = (pe_action_t ) iter->data;

	if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	runnable = false;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	pe_action_t *promote = promote_action(rsc, node, optional);

	pe_rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node));

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the promote becomes necessary for
	* other reasons later.
	*/
	pe_rsc_trace(rsc,
	"Promotion of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pe__set_action_flags(promote, pcmk_action_pseudo);
	}
	} else {
	pe_rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
	rsc->id, pe__node_name(node));
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
	true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t promote = (pe_action_t ) iter->data;

	pe__clear_action_flags(promote, pcmk_action_runnable);
	}
	g_list_free(action_list);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to demote a resource wherever it is active
	*
	* \param[in,out] rsc Resource being demoted
	* \param[in] node Node where resource should be demoted (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	demote_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	/* Since this will only be called for a primitive (possibly as an instance
	* of a collective resource), the resource is multiply active if it is
	* running on more than one node, so we want to demote on all of them as
	* part of recovery, regardless of which one is the desired node.
	*/
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t current = (pe_node_t ) iter->data;

	if (is_expected_node(rsc, current)) {
	pe_rsc_trace(rsc,
	"Skipping demote of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	} else {
	pe_rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(current));
	demote_action(rsc, current, optional);
	}
	}
	}

	static void
	assert_role_error(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	CRM_ASSERT(false);
	}

	/*!
	* \internal
	* \brief Schedule cleanup of a resource
	*
	* \param[in,out] rsc Resource to clean up
	* \param[in] node Node to clean up on
	* \param[in] optional Whether clean-up should be optional
	*/
	void
	pcmk__schedule_cleanup(pe_resource_t rsc, const pe_node_t node, bool optional)
	{
	/* If the cleanup is required, its orderings are optional, because they're
	* relevant only if both actions are required. Conversely, if the cleanup is
	* optional, the orderings make the then action required if the first action
	* becomes required.
	*/
	- uint32_t flag = optional? pe_order_implies_then : pcmk__ar_ordered;
	+ uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;

	CRM_CHECK((rsc != NULL) && (node != NULL), return);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
	rsc->id, pe__node_name(node));
	return;
	}

	if (node->details->unclean \|\| !node->details->online) {
	pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
	rsc->id, pe__node_name(node));
	return;
	}

	crm_notice("Scheduling clean-up of %s on %s", rsc->id, pe__node_name(node));
	delete_action(rsc, node, optional);

	// stop -> clean-up -> start
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_DELETE, flag);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
	rsc, PCMK_ACTION_START, flag);
	}

	/*!
	* \internal
	* \brief Add primitive meta-attributes relevant to graph actions to XML
	*
	* \param[in] rsc Primitive resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__primitive_add_graph_meta(const pe_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;
	char *value = NULL;
	const pe_resource_t *parent = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (xml != NULL));

	/* Clone instance numbers get set internally as meta-attributes, and are
	* needed in the transition graph (for example, to tell unique clone
	* instances apart).
	*/
	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	if (value != NULL) {
	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	// Not sure if this one is really needed ...
	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value != NULL) {
	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	/* The container meta-attribute can be set on the primitive itself or one of
	* its parents (for example, a group inside a container resource), so check
	* them all, and keep the highest one found.
	*/
	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container != NULL) {
	crm_xml_add(xml, CRM_META "_" XML_RSC_ATTR_CONTAINER,
	parent->container->id);
	}
	}

	/* Bundle replica children will get their external-ip set internally as a
	* meta-attribute. The graph action needs it, but under a different naming
	* convention than other meta-attributes.
	*/
	value = g_hash_table_lookup(rsc->meta, "external-ip");
	if (value != NULL) {
	crm_xml_add(xml, "pcmk_external_ip", value);
	}
	}

	// Primitive implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__primitive_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}

	/*!
	* \internal
	* \brief Get epoch time of node's shutdown attribute (or now if none)
	*
	* \param[in,out] node Node to check
	*
	* \return Epoch time corresponding to shutdown attribute if set or now if not
	*/
	static time_t
	shutdown_time(pe_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown != NULL) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return (result == 0)? get_effective_time(node->details->data_set) : result;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it's not locked to the node
	*
	* \param[in] data Node to check
	* \param[in,out] user_data Resource to check
	*/
	static void
	ban_if_not_locked(gpointer data, gpointer user_data)
	{
	const pe_node_t node = (const pe_node_t ) data;
	pe_resource_t rsc = (pe_resource_t ) user_data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, rsc->cluster);
	}
	}

	// Primitive implementation of resource_alloc_functions_t:shutdown_lock()
	void
	pcmk__primitive_shutdown_lock(pe_resource_t *rsc)
	{
	const char *class = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Fence devices and remote connections can't be locked
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, pe__node_name(node));
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (rsc->cluster->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, pe__node_name(rsc->lock_node),
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, rsc->cluster);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, pe__node_name(rsc->lock_node));
	}

	// If resource is locked to one node, ban it from all other nodes
	g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index 0e0dab84a1..1438bff079 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,902 +1,902 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pe_action_t probe, const pe_resource_t rsc,
	const pe_node_t *node)
	{
	// Check whether resource is currently active on node
	pe_node_t *running = pe_find_node_id(rsc->running_on, node->details->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pe_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pe_resource_t rsc1, pe_resource_t rsc2)
	{
	if ((rsc1->allocated_to != NULL)
	&& (g_hash_table_lookup(rsc1->known_on,
	rsc1->allocated_to->details->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pe_node_t *node)
	{
	const pe_resource_t *guest_rsc = node->details->remote_rsc->container;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk_rsc_failed)
	\|\| (guest_rsc->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->role > pcmk_role_stopped)
	&& (guest_rsc->allocated_to != NULL)
	&& (pe_find_node(guest_rsc->running_on,
	guest_rsc->allocated_to->details->uname) == NULL));
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pe_action_t *
	probe_action(pe_resource_t rsc, pe_node_t node)
	{
	pe_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	role2text(rsc->role), rsc->id, pe__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE, TRUE,
	rsc->cluster);
	pe__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pe_resource_t rsc, pe_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pe_action_t *probe = NULL;
	pe_node_t *allowed = NULL;
	pe_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pe__is_guest_or_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(rsc->cluster, rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (rsc->is_remote_node) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->children != NULL) {
	return pcmk__probe_resource_list(rsc->children, node);
	}

	if ((rsc->container != NULL) && !rsc->is_remote_node) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	} else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	if (allowed->rsc_discover_mode == pcmk_probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pe__is_guest_node(node)) {
	pe_resource_t *guest = node->details->remote_rsc->container;

	if (guest->role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pe_rsc_is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->running_on == NULL)) {
	pe__set_order_flags(flags, pe_order_runnable_left);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->cluster);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->cluster);

	return true;

	no_probe:
	pe_rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->details->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pe_action_t probe, const pe_action_t then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& pe__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta, "stonith_action");

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	&& !pe__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] data_set Cluster working set
	*/
	static void
	add_probe_orderings_for_stops(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pe__ordering_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pe_action_t *first = NULL;
	pe_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->lh_rsc == NULL) \|\| (order->lh_rsc == order->rh_rsc)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->lh_action;
	then = order->rh_action;
	if (((first == NULL) && (order->lh_action_task == NULL))
	\|\| ((then == NULL) && (order->rh_action_task == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->lh_action_task,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* container. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the container starts again.
	*/
	if ((order->rh_rsc != NULL)
	&& (order->lh_rsc->container == order->rh_rsc)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	&& pcmk__ends_with(order->rh_action_task,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pe__set_order_flags(order_flags, pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pe_order_same_node)) {
	pe__set_order_flags(order_flags, pe_order_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pe_order_anti_colocation)
	\|\| (order->flags == pe_order_load)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->lh_rsc, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rh_rsc != NULL) {
	then_actions = find_actions(order->rh_rsc->actions,
	order->rh_action_task, NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->lh_action_task : first->uuid),
	((then == NULL)? order->rh_action_task : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pe_action_t probe = (pe_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pe_action_t then = (pe_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pe_action_t probe, pe_action_wrapper_t after)
	{
	uint32_t flags = pcmk__ar_ordered\|pe_order_runnable_left;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pe_order_runnable_left` order type for that as long as
	* any of the clone instances are running to prevent them from being
	* unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->variant <= pcmk_rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pe_order_runnable_left)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->action->uuid, pe__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pe_action_wrapper_t then = (pe_action_wrapper_t ) then_iter->data;

	if (then->action->rsc->running_on
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pe__node_name(probe->node),
	then->action->uuid, pe__node_name(then->action->node), flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pe_action_t probe, pe_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pe_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| (probe->rsc == NULL)
	\|\| (probe->rsc->variant != pcmk_rsc_variant_primitive)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pe__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->uuid, pe__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant == pcmk_rsc_variant_primitive)
	&& (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pe_action_t then = (pe_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant > pcmk_rsc_variant_group)) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) iter->data;

	- /* pe_order_implies_then is the reason why a required A.start
	+ /* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pe_order_implies_then_on_node though. It's now only
	* used for unfencing case, which tends to introduce transition
	* loops...
	*/
	- if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
	+ if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pe_order_implies_first_printed \| pe_order_runnable_left
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->variant < pcmk_rsc_variant_group)
	\|\| (probe->rsc->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)
	\|\| (after_wrapper->action->rsc->variant > pcmk_rsc_variant_group)
	\|\| (after->rsc != after_wrapper->action->rsc->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->variant > pcmk_rsc_variant_group) && interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != after_wrapper->action->rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pe__node_name(after->node),
	after_wrapper->action->uuid,
	pe__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] data_set Cluster working set
	*/
	static void
	clear_actions_tracking_flag(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t *action = iter->data;

	pe__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pe_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, add_start_restart_orderings_for_rsc,
	NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pe_action_t probe = (pe_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pe_action_wrapper_t then = (pe_action_wrapper_t ) then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->cluster);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] data_set Cluster working set
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pe_working_set_t *data_set)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	pe_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pe_action_wrapper_t before = (pe_action_wrapper_t ) actions->data;

	pe_action_t *first = before->action;
	pe_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = (pe_action_wrapper_t *) clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pe_rsc_is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Applying %s before %s %d", first->uuid, start->uuid,
	pe__const_top_resource(first_rsc, false)->variant);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pe_action_t probe = (pe_action_t ) probe_iter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pe_working_set_t *data_set)
	{
	// Add orderings for "probe then X"
	g_list_foreach(data_set->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(data_set);

	order_then_probes(data_set);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] data_set Cluster working set
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pe_working_set_t *data_set)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;
	const char *probed = NULL;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(data_set, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) { // ... or nodes that need fencing
	continue;

	} else if (!node->details->rsc_discovery_enabled) {
	// The user requested that probes not be done on this node
	continue;
	}

	/* This is no longer needed for live clusters, since the probe_complete
	* node attribute will never be in the CIB. However this is still useful
	* for processing old saved CIBs (< 1.1.14), including the
	* reprobe-target_rc regression test.
	*/
	probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pe_action_t *probe_op = NULL;

	probe_op = custom_action(NULL,
	crm_strdup_printf("%s-%s", CRM_OP_REPROBE,
	node->details->uname),
	CRM_OP_REPROBE, node, FALSE, TRUE,
	data_set);
	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT,
	XML_BOOLEAN_TRUE);
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(data_set->resources, node);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_recurring.c b/lib/pacemaker/pcmk_sched_recurring.c
	index 2850a0d329..f8cf010065 100644
	--- a/lib/pacemaker/pcmk_sched_recurring.c
	+++ b/lib/pacemaker/pcmk_sched_recurring.c
	@@ -1,718 +1,719 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Information parsed from an operation history entry in the CIB
	struct op_history {
	// XML attributes
	const char *id; // ID of history entry
	const char *name; // Action name

	// Parsed information
	char *key; // Operation key for action
	enum rsc_role_e role; // Action role (or pcmk_role_unknown for default)
	guint interval_ms; // Action interval
	};

	/*!
	* \internal
	* \brief Parse an interval from XML
	*
	* \param[in] xml XML containing an interval attribute
	*
	* \return Interval parsed from XML (or 0 as default)
	*/
	static guint
	xe_interval(const xmlNode *xml)
	{
	return crm_parse_interval_spec(crm_element_value(xml,
	XML_LRM_ATTR_INTERVAL));
	}

	/*!
	* \internal
	* \brief Check whether an operation exists multiple times in resource history
	*
	* \param[in] rsc Resource with history to search
	* \param[in] name Name of action to search for
	* \param[in] interval_ms Interval (in milliseconds) of action to search for
	*
	* \return true if an operation with \p name and \p interval_ms exists more than
	* once in the operation history of \p rsc, otherwise false
	*/
	static bool
	is_op_dup(const pe_resource_t rsc, const char name, guint interval_ms)
	{
	const char *id = NULL;

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	// Check whether action name and interval match
	if (!pcmk__str_eq(crm_element_value(op, "name"), name, pcmk__str_none)
	\|\| (xe_interval(op) != interval_ms)) {
	continue;
	}

	if (ID(op) == NULL) {
	continue; // Shouldn't be possible
	}

	if (id == NULL) {
	id = ID(op); // First matching op
	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(op), id);
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether an action name is one that can be recurring
	*
	* \param[in] name Action name to check
	*
	* \return true if \p name is an action known to be unsuitable as a recurring
	* operation, otherwise false
	*
	* \note Pacemaker's current philosophy is to allow users to configure recurring
	* operations except for a short list of actions known not to be suitable
	* for that (as opposed to allowing only actions known to be suitable,
	* which includes only monitor). Among other things, this approach allows
	* users to define their own custom operations and make them recurring,
	* though that use case is not well tested.
	*/
	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__str_any_of(name, PCMK_ACTION_STOP, PCMK_ACTION_START,
	PCMK_ACTION_DEMOTE, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_RELOAD_AGENT,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL);
	}

	/*!
	* \internal
	* \brief Check whether a resource history entry is for a recurring action
	*
	* \param[in] rsc Resource that history entry is for
	* \param[in] xml XML of resource history entry to check
	* \param[out] op Where to store parsed info if recurring
	*
	* \return true if \p xml is for a recurring action, otherwise false
	*/
	static bool
	is_recurring_history(const pe_resource_t rsc, const xmlNode xml,
	struct op_history *op)
	{
	const char *role = NULL;

	op->interval_ms = xe_interval(xml);
	if (op->interval_ms == 0) {
	return false; // Not recurring
	}

	op->id = ID(xml);
	if (pcmk__str_empty(op->id)) {
	pcmk__config_err("Ignoring resource history entry without ID");
	return false; // Shouldn't be possible (unless CIB was manually edited)
	}

	op->name = crm_element_value(xml, "name");
	if (op_cannot_recur(op->name)) {
	pcmk__config_err("Ignoring %s because %s action cannot be recurring",
	op->id, pcmk__s(op->name, "unnamed"));
	return false;
	}

	// There should only be one recurring operation per action/interval
	if (is_op_dup(rsc, op->name, op->interval_ms)) {
	return false;
	}

	// Ensure role is valid if specified
	role = crm_element_value(xml, "role");
	if (role == NULL) {
	op->role = pcmk_role_unknown;
	} else {
	op->role = text2role(role);
	if (op->role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring %s because %s is not a valid role",
	op->id, role);
	}
	}

	// Disabled resources don't get monitored
	op->key = pcmk__op_key(rsc->id, op->name, op->interval_ms);
	if (find_rsc_op_entry(rsc, op->key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	op->id, rsc->id);
	free(op->key);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a recurring action for an active role should be optional
	*
	* \param[in] rsc Resource that recurring action is for
	* \param[in] node Node that \p rsc will be active on (if any)
	* \param[in] key Operation key for recurring action to check
	* \param[in,out] start Start action for \p rsc
	*
	* \return true if recurring action should be optional, otherwise false
	*/
	static bool
	active_recurring_should_be_optional(const pe_resource_t *rsc,
	const pe_node_t node, const char key,
	pe_action_t *start)
	{
	GList *possible_matches = NULL;

	if (node == NULL) { // Should only be possible if unmanaged and stopped
	pe_rsc_trace(rsc, "%s will be mandatory because resource is unmanaged",
	key);
	return false;
	}

	if (!pcmk_is_set(rsc->cmds->action_flags(start, NULL),
	pcmk_action_optional)) {
	pe_rsc_trace(rsc, "%s will be mandatory because %s is",
	key, start->uuid);
	return false;
	}

	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "%s will be mandatory because it is not active on %s",
	key, pe__node_name(node));
	return false;
	}

	for (const GList *iter = possible_matches;
	iter != NULL; iter = iter->next) {

	const pe_action_t op = (const pe_action_t ) iter->data;

	if (pcmk_is_set(op->flags, pcmk_action_reschedule)) {
	pe_rsc_trace(rsc,
	"%s will be mandatory because "
	"it needs to be rescheduled", key);
	g_list_free(possible_matches);
	return false;
	}
	}

	g_list_free(possible_matches);
	return true;
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for an active role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in,out] start Start action for \p rsc on \p node
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_active(pe_resource_t rsc, pe_action_t start,
	const pe_node_t node, const struct op_history op)
	{
	pe_action_t *mon = NULL;
	bool is_optional = true;
	const bool is_default_role = (op->role == pcmk_role_unknown);

	// We're only interested in recurring actions for active roles
	if (op->role == pcmk_role_stopped) {
	return;
	}

	is_optional = active_recurring_should_be_optional(rsc, node, op->key,
	start);

	if ((!is_default_role && (rsc->next_role != op->role))
	\|\| (is_default_role && (rsc->next_role == pcmk_role_promoted))) {
	// Configured monitor role doesn't match role resource will have

	if (is_optional) { // It's running, so cancel it
	char *after_key = NULL;
	pe_action_t *cancel_op = pcmk__new_cancel_action(rsc, op->name,
	op->interval_ms,
	node);

	switch (rsc->role) {
	case pcmk_role_unpromoted:
	case pcmk_role_started:
	if (rsc->next_role == pcmk_role_promoted) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == pcmk_role_stopped) {
	after_key = stop_key(rsc);
	}

	break;
	case pcmk_role_promoted:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	pcmk__new_ordering(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, rsc->cluster);
	}
	}

	do_crm_log((is_optional? LOG_INFO : LOG_TRACE),
	"%s recurring action %s because %s configured for %s role "
	"(not %s)",
	(is_optional? "Cancelling" : "Ignoring"), op->key, op->id,
	role2text(is_default_role? pcmk_role_unpromoted : op->role),
	role2text(rsc->next_role));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s %s on %s)",
	(is_optional? "optional" : "mandatory"), op->key,
	op->id, rsc->id, role2text(rsc->next_role),
	pe__node_name(node));

	mon = custom_action(rsc, strdup(op->key), op->name, node, is_optional, TRUE,
	rsc->cluster);

	if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	pe_rsc_trace(rsc, "%s is unrunnable because start is", mon->uuid);
	pe__clear_action_flags(mon, pcmk_action_runnable);

	} else if ((node == NULL) \|\| !node->details->online
	\|\| node->details->unclean) {
	pe_rsc_trace(rsc, "%s is unrunnable because no node is available",
	mon->uuid);
	pe__clear_action_flags(mon, pcmk_action_runnable);

	} else if (!pcmk_is_set(mon->flags, pcmk_action_optional)) {
	pe_rsc_info(rsc, "Start %s-interval %s for %s on %s",
	pcmk__readable_interval(op->interval_ms), mon->task,
	rsc->id, pe__node_name(node));
	}

	if (rsc->next_role == pcmk_role_promoted) {
	pe__add_action_expected_result(mon, CRM_EX_PROMOTED);
	}

	// Order monitor relative to other actions
	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__new_ordering(rsc, start_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	- pe_order_implies_then\|pe_order_runnable_left,
	+ pcmk__ar_first_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	pcmk__new_ordering(rsc, reload_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	- pe_order_implies_then\|pe_order_runnable_left,
	+ pcmk__ar_first_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	if (rsc->next_role == pcmk_role_promoted) {
	pcmk__new_ordering(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pcmk__ar_ordered\|pe_order_runnable_left,
	rsc->cluster);

	} else if (rsc->role == pcmk_role_promoted) {
	pcmk__new_ordering(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pcmk__ar_ordered\|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief Cancel a recurring action if running on a node
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node to cancel action on
	* \param[in] key Operation key for action
	* \param[in] name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	*/
	static void
	cancel_if_running(pe_resource_t rsc, const pe_node_t node, const char *key,
	const char *name, guint interval_ms)
	{
	GList *possible_matches = find_actions_exact(rsc->actions, key, node);
	pe_action_t *cancel_op = NULL;

	if (possible_matches == NULL) {
	return; // Recurring action isn't running on this node
	}
	g_list_free(possible_matches);

	cancel_op = pcmk__new_cancel_action(rsc, name, interval_ms, node);

	switch (rsc->next_role) {
	case pcmk_role_started:
	case pcmk_role_unpromoted:
	/* Order starts after cancel. If the current role is
	* stopped, this cancels the monitor before the resource
	* starts; if the current role is started, then this cancels
	* the monitor on a migration target before starting there.
	*/
	pcmk__new_ordering(rsc, NULL, cancel_op,
	rsc, start_key(rsc), NULL,
	pe_order_runnable_left, rsc->cluster);
	break;
	default:
	break;
	}
	pe_rsc_info(rsc,
	"Cancelling %s-interval %s action for %s on %s because "
	"configured for " PCMK__ROLE_STOPPED " role (not %s)",
	pcmk__readable_interval(interval_ms), name, rsc->id,
	pe__node_name(node), role2text(rsc->next_role));
	}

	/*!
	* \internal
	* \brief Order an action after all probes of a resource on a node
	*
	* \param[in,out] rsc Resource to check for probes
	* \param[in] node Node to check for probes of \p rsc
	* \param[in,out] action Action to order after probes of \p rsc on \p node
	*/
	static void
	order_after_probes(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *probes = pe__resource_actions(rsc, node, PCMK_ACTION_MONITOR, FALSE);

	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	order_actions((pe_action_t *) iter->data, action,
	pe_order_runnable_left);
	}
	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Order an action after all stops of a resource on a node
	*
	* \param[in,out] rsc Resource to check for stops
	* \param[in] node Node to check for stops of \p rsc
	* \param[in,out] action Action to order after stops of \p rsc on \p node
	*/
	static void
	order_after_stops(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *stop_ops = pe__resource_actions(rsc, node, PCMK_ACTION_STOP, TRUE);

	for (GList *iter = stop_ops; iter != NULL; iter = iter->next) {
	pe_action_t stop = (pe_action_t ) iter->data;

	if (!pcmk_is_set(stop->flags, pcmk_action_optional)
	&& !pcmk_is_set(action->flags, pcmk_action_optional)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc, "%s optional on %s: unmanaged",
	action->uuid, pe__node_name(node));
	pe__set_action_flags(action, pcmk_action_optional);
	}

	if (!pcmk_is_set(stop->flags, pcmk_action_runnable)) {
	crm_debug("%s unrunnable on %s: stop is unrunnable",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pcmk_action_runnable);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__new_ordering(rsc, stop_key(rsc), stop,
	NULL, NULL, action,
	- pe_order_implies_then\|pe_order_runnable_left,
	+ pcmk__ar_first_implies_then
	+ \|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	g_list_free(stop_ops);
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for inactive role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_inactive(pe_resource_t rsc, const pe_node_t node,
	const struct op_history *op)
	{
	GList *possible_matches = NULL;

	// We're only interested in recurring actions for the inactive role
	if (op->role != pcmk_role_stopped) {
	return;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for " PCMK__ROLE_STOPPED
	" role are not supported for anonymous clones)", op->id);
	return; // @TODO add support
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on nodes "
	"where it should not be running", op->id, rsc->id);

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pe_node_t stop_node = (pe_node_t ) iter->data;

	bool is_optional = true;
	pe_action_t *stopped_mon = NULL;

	// Cancel action on node where resource will be active
	if ((node != NULL)
	&& pcmk__str_eq(stop_node->details->uname, node->details->uname,
	pcmk__str_casei)) {
	cancel_if_running(rsc, node, op->key, op->name, op->interval_ms);
	continue;
	}

	// Recurring action on this node is optional if it's already active here
	possible_matches = find_actions_exact(rsc->actions, op->key, stop_node);
	is_optional = (possible_matches != NULL);
	g_list_free(possible_matches);

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s "
	PCMK__ROLE_STOPPED " on %s)",
	(is_optional? "optional" : "mandatory"),
	op->key, op->id, rsc->id, pe__node_name(stop_node));

	stopped_mon = custom_action(rsc, strdup(op->key), op->name, stop_node,
	is_optional, TRUE, rsc->cluster);

	pe__add_action_expected_result(stopped_mon, CRM_EX_NOT_RUNNING);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	order_after_probes(rsc, stop_node, stopped_mon);
	}

	/* The recurring action is for the inactive role, so it shouldn't be
	* performed until the resource is inactive.
	*/
	order_after_stops(rsc, stop_node, stopped_mon);

	if (!stop_node->details->online \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s unrunnable on %s: node unavailable)",
	stopped_mon->uuid, pe__node_name(stop_node));
	pe__clear_action_flags(stopped_mon, pcmk_action_runnable);
	}

	if (pcmk_is_set(stopped_mon->flags, pcmk_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pcmk_action_optional)) {
	crm_notice("Start recurring %s-interval %s for "
	PCMK__ROLE_STOPPED " %s on %s",
	pcmk__readable_interval(op->interval_ms),
	stopped_mon->task, rsc->id, pe__node_name(stop_node));
	}
	}
	}

	/*!
	* \internal
	* \brief Create recurring actions for a resource
	*
	* \param[in,out] rsc Resource to create recurring actions for
	*/
	void
	pcmk__create_recurring_actions(pe_resource_t *rsc)
	{
	pe_action_t *start = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for blocked resource %s",
	rsc->id);
	return;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for %s "
	"in maintenance mode", rsc->id);
	return;
	}

	if (rsc->allocated_to == NULL) {
	// Recurring actions for active roles not needed

	} else if (rsc->allocated_to->details->maintenance) {
	pe_rsc_trace(rsc,
	"Skipping recurring actions for %s on %s "
	"in maintenance mode",
	rsc->id, pe__node_name(rsc->allocated_to));

	} else if ((rsc->next_role != pcmk_role_stopped)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	// Recurring actions for active roles needed
	start = start_action(rsc, rsc->allocated_to, TRUE);
	}

	pe_rsc_trace(rsc, "Creating any recurring actions needed for %s", rsc->id);

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	struct op_history op_history = { NULL, };

	if (!is_recurring_history(rsc, op, &op_history)) {
	continue;
	}

	if (start != NULL) {
	recurring_op_for_active(rsc, start, rsc->allocated_to, &op_history);
	}
	recurring_op_for_inactive(rsc, rsc->allocated_to, &op_history);

	free(op_history.key);
	}
	}

	/*!
	* \internal
	* \brief Create an executor cancel action
	*
	* \param[in,out] rsc Resource of action to cancel
	* \param[in] task Name of action to cancel
	* \param[in] interval_ms Interval of action to cancel
	* \param[in] node Node of action to cancel
	*
	* \return Created op
	*/
	pe_action_t *
	pcmk__new_cancel_action(pe_resource_t rsc, const char task, guint interval_ms,
	const pe_node_t *node)
	{
	pe_action_t *cancel_op = NULL;
	char *key = NULL;
	char *interval_ms_s = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL) && (node != NULL));

	// @TODO dangerous if possible to schedule another action with this key
	key = pcmk__op_key(rsc->id, task, interval_ms);

	cancel_op = custom_action(rsc, key, PCMK_ACTION_CANCEL, node, FALSE, TRUE,
	rsc->cluster);

	pcmk__str_update(&cancel_op->task, PCMK_ACTION_CANCEL);
	pcmk__str_update(&cancel_op->cancel_task, task);

	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
	free(interval_ms_s);

	return cancel_op;
	}

	/*!
	* \internal
	* \brief Schedule cancellation of a recurring action
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] call_id Action's call ID from history
	* \param[in] task Action name
	* \param[in] interval_ms Action interval
	* \param[in] node Node that history entry is for
	* \param[in] reason Short description of why action is cancelled
	*/
	void
	pcmk__schedule_cancel(pe_resource_t rsc, const char call_id, const char *task,
	guint interval_ms, const pe_node_t *node,
	const char *reason)
	{
	pe_action_t *cancel = NULL;

	CRM_CHECK((rsc != NULL) && (task != NULL)
	&& (node != NULL) && (reason != NULL),
	return);

	crm_info("Recurring %s-interval %s for %s will be stopped on %s: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node), reason);
	cancel = pcmk__new_cancel_action(rsc, task, interval_ms, node);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);

	// Cancellations happen after stops
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, rsc, NULL, cancel,
	pcmk__ar_ordered, rsc->cluster);
	}

	/*!
	* \internal
	* \brief Create a recurring action marked as needing rescheduling if active
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action being rescheduled
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where action should be rescheduled
	*/
	void
	pcmk__reschedule_recurring(pe_resource_t rsc, const char task,
	guint interval_ms, pe_node_t *node)
	{
	pe_action_t *op = NULL;

	trigger_unfencing(rsc, node, "Device parameters changed (reschedule)",
	NULL, rsc->cluster);
	op = custom_action(rsc, pcmk__op_key(rsc->id, task, interval_ms),
	task, node, TRUE, TRUE, rsc->cluster);
	pe__set_action_flags(op, pcmk_action_reschedule);
	}

	/*!
	* \internal
	* \brief Check whether an action is recurring
	*
	* \param[in] action Action to check
	*
	* \return true if \p action has a nonzero interval, otherwise false
	*/
	bool
	pcmk__action_is_recurring(const pe_action_t *action)
	{
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return false;
	}
	return (interval_ms > 0);
	}
	diff --git a/lib/pacemaker/pcmk_sched_remote.c b/lib/pacemaker/pcmk_sched_remote.c
	index 443efd5992..af6aba8197 100644
	--- a/lib/pacemaker/pcmk_sched_remote.c
	+++ b/lib/pacemaker/pcmk_sched_remote.c
	@@ -1,725 +1,726 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum remote_connection_state {
	remote_state_unknown = 0,
	remote_state_alive = 1,
	remote_state_resting = 2,
	remote_state_failed = 3,
	remote_state_stopped = 4
	};

	static const char *
	state2text(enum remote_connection_state state)
	{
	switch (state) {
	case remote_state_unknown:
	return "unknown";
	case remote_state_alive:
	return "alive";
	case remote_state_resting:
	return "resting";
	case remote_state_failed:
	return "failed";
	case remote_state_stopped:
	return "stopped";
	}

	return "impossible";
	}

	/* We always use pe_order_preserve with these convenience functions to exempt
	* internally generated constraints from the prohibition of user constraints
	* involving remote connection resources.
	*
	* The start ordering additionally uses pe_order_runnable_left so that the
	* specified action is not runnable if the start is not runnable.
	*/

	static inline void
	order_start_then_action(pe_resource_t first_rsc, pe_action_t then_action,
	uint32_t extra)
	{
	if ((first_rsc != NULL) && (then_action != NULL)) {
	pcmk__new_ordering(first_rsc, start_key(first_rsc), NULL,
	then_action->rsc, NULL, then_action,
	pe_order_preserve\|pe_order_runnable_left\|extra,
	first_rsc->cluster);
	}
	}

	static inline void
	order_action_then_stop(pe_action_t first_action, pe_resource_t then_rsc,
	uint32_t extra)
	{
	if ((first_action != NULL) && (then_rsc != NULL)) {
	pcmk__new_ordering(first_action->rsc, NULL, first_action,
	then_rsc, stop_key(then_rsc), NULL,
	pe_order_preserve\|extra, then_rsc->cluster);
	}
	}

	static enum remote_connection_state
	get_remote_node_state(const pe_node_t *node)
	{
	const pe_resource_t *remote_rsc = NULL;
	const pe_node_t *cluster_node = NULL;

	CRM_ASSERT(node != NULL);

	remote_rsc = node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	cluster_node = pe__current_node(remote_rsc);

	/* If the cluster node the remote connection resource resides on
	* is unclean or went offline, we can't process any operations
	* on that remote node until after it starts elsewhere.
	*/
	if ((remote_rsc->next_role == pcmk_role_stopped)
	\|\| (remote_rsc->allocated_to == NULL)) {

	// The connection resource is not going to run anywhere

	if ((cluster_node != NULL) && cluster_node->details->unclean) {
	/* The remote connection is failed because its resource is on a
	* failed node and can't be recovered elsewhere, so we must fence.
	*/
	return remote_state_failed;
	}

	if (!pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)) {
	/* Connection resource is cleanly stopped */
	return remote_state_stopped;
	}

	/* Connection resource is failed */

	if ((remote_rsc->next_role == pcmk_role_stopped)
	&& remote_rsc->remote_reconnect_ms
	&& node->details->remote_was_fenced
	&& !pe__shutdown_requested(node)) {

	/* We won't know whether the connection is recoverable until the
	* reconnect interval expires and we reattempt connection.
	*/
	return remote_state_unknown;
	}

	/* The remote connection is in a failed state. If there are any
	* resources known to be active on it (stop) or in an unknown state
	* (probe), we must assume the worst and fence it.
	*/
	return remote_state_failed;

	} else if (cluster_node == NULL) {
	/* Connection is recoverable but not currently running anywhere, so see
	* if we can recover it first
	*/
	return remote_state_unknown;

	} else if (cluster_node->details->unclean
	\|\| !(cluster_node->details->online)) {
	// Connection is running on a dead node, see if we can recover it first
	return remote_state_resting;

	} else if (pcmk__list_of_multiple(remote_rsc->running_on)
	&& (remote_rsc->partial_migration_source != NULL)
	&& (remote_rsc->partial_migration_target != NULL)) {
	/* We're in the middle of migrating a connection resource, so wait until
	* after the migration completes before performing any actions.
	*/
	return remote_state_resting;

	}
	return remote_state_alive;
	}

	/*!
	* \internal
	* \brief Order actions on remote node relative to actions for the connection
	*
	* \param[in,out] action An action scheduled on a Pacemaker Remote node
	*/
	static void
	apply_remote_ordering(pe_action_t *action)
	{
	pe_resource_t *remote_rsc = NULL;
	enum action_tasks task = text2task(action->task);
	enum remote_connection_state state = get_remote_node_state(action->node);

	uint32_t order_opts = pcmk__ar_none;

	if (action->rsc == NULL) {
	return;
	}

	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	crm_trace("Order %s action %s relative to %s%s (state: %s)",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "",
	remote_rsc->id, state2text(state));

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Migration ops map to pcmk_action_unspecified, but we need to apply
	* the same ordering as for stop or demote (see get_router_node()).
	*/
	task = pcmk_action_stop;
	}

	switch (task) {
	case pcmk_action_start:
	case pcmk_action_promote:
	order_opts = pcmk__ar_none;

	if (state == remote_state_failed) {
	/* Force recovery, by making this action required */
	- pe__set_order_flags(order_opts, pe_order_implies_then);
	+ pe__set_order_flags(order_opts, pcmk__ar_first_implies_then);
	}

	/* Ensure connection is up before running this action */
	order_start_then_action(remote_rsc, action, order_opts);
	break;

	case pcmk_action_stop:
	if (state == remote_state_alive) {
	order_action_then_stop(action, remote_rsc,
	pcmk__ar_then_implies_first);

	} else if (state == remote_state_failed) {
	/* The resource is active on the node, but since we don't have a
	* valid connection, the only way to stop the resource is by
	* fencing the node. There is no need to order the stop relative
	* to the remote connection, since the stop will become implied
	* by the fencing.
	*/
	pe_fence_node(remote_rsc->cluster, action->node,
	"resources are active but "
	"connection is unrecoverable",
	FALSE);

	} else if (remote_rsc->next_role == pcmk_role_stopped) {
	/* State must be remote_state_unknown or remote_state_stopped.
	* Since the connection is not coming back up in this
	* transition, stop this resource first.
	*/
	order_action_then_stop(action, remote_rsc,
	pcmk__ar_then_implies_first);

	} else {
	/* The connection is going to be started somewhere else, so
	* stop this resource after that completes.
	*/
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	break;

	case pcmk_action_demote:
	/* Only order this demote relative to the connection start if the
	* connection isn't being torn down. Otherwise, the demote would be
	* blocked because the connection start would not be allowed.
	*/
	if ((state == remote_state_resting)
	\|\| (state == remote_state_unknown)) {

	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	} /* Otherwise we can rely on the stop ordering */
	break;

	default:
	/* Wait for the connection resource to be up */
	if (pcmk__action_is_recurring(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	order_start_then_action(remote_rsc, action,
	- pe_order_implies_then);
	+ pcmk__ar_first_implies_then);

	} else {
	pe_node_t *cluster_node = pe__current_node(remote_rsc);

	if ((task == pcmk_action_monitor) && (state == remote_state_failed)) {
	/* We would only be here if we do not know the state of the
	* resource on the remote node. Since we have no way to find
	* out, it is necessary to fence the node.
	*/
	pe_fence_node(remote_rsc->cluster, action->node,
	"resources are in unknown state "
	"and connection is unrecoverable", FALSE);
	}

	if ((cluster_node != NULL) && (state == remote_state_stopped)) {
	/* The connection is currently up, but is going down
	* permanently. Make sure we check services are actually
	* stopped _before_ we let the connection get closed.
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_runnable_left);

	} else {
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	}
	break;
	}
	}

	static void
	apply_container_ordering(pe_action_t *action)
	{
	/* VMs are also classified as containers for these purposes... in
	* that they both involve a 'thing' running on a real or remote
	* cluster node.
	*
	* This allows us to be smarter about the type and extent of
	* recovery actions required in various scenarios
	*/
	pe_resource_t *remote_rsc = NULL;
	pe_resource_t *container = NULL;
	enum action_tasks task = text2task(action->task);

	CRM_ASSERT(action->rsc != NULL);
	CRM_ASSERT(action->node != NULL);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc != NULL);

	container = remote_rsc->container;
	CRM_ASSERT(container != NULL);

	if (pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	pe_fence_node(action->rsc->cluster, action->node, "container failed",
	FALSE);
	}

	crm_trace("Order %s action %s relative to %s%s for %s%s",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pcmk_rsc_failed)? "failed " : "",
	remote_rsc->id,
	pcmk_is_set(container->flags, pcmk_rsc_failed)? "failed " : "",
	container->id);

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Migration ops map to pcmk_action_unspecified, but we need to apply
	* the same ordering as for stop or demote (see get_router_node()).
	*/
	task = pcmk_action_stop;
	}

	switch (task) {
	case pcmk_action_start:
	case pcmk_action_promote:
	// Force resource recovery if the container is recovered
	- order_start_then_action(container, action, pe_order_implies_then);
	+ order_start_then_action(container, action,
	+ pcmk__ar_first_implies_then);

	// Wait for the connection resource to be up, too
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	break;

	case pcmk_action_stop:
	case pcmk_action_demote:
	if (pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	/* When the container representing a guest node fails, any stop
	* or demote actions for resources running on the guest node
	* are implied by the container stopping. This is similar to
	* how fencing operations work for cluster nodes and remote
	* nodes.
	*/
	} else {
	/* Ensure the operation happens before the connection is brought
	* down.
	*
	* If we really wanted to, we could order these after the
	* connection start, IFF the container's current role was
	* stopped (otherwise we re-introduce an ordering loop when the
	* connection is restarting).
	*/
	order_action_then_stop(action, remote_rsc, pcmk__ar_none);
	}
	break;

	default:
	/* Wait for the connection resource to be up */
	if (pcmk__action_is_recurring(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	if (task != pcmk_action_unspecified) {
	order_start_then_action(remote_rsc, action,
	- pe_order_implies_then);
	+ pcmk__ar_first_implies_then);
	}
	} else {
	order_start_then_action(remote_rsc, action, pcmk__ar_none);
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Order all relevant actions relative to remote connection actions
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__order_remote_connection_actions(pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pcmk_sched_have_remote_nodes)) {
	return;
	}

	crm_trace("Creating remote connection orderings");

	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t *action = iter->data;
	pe_resource_t *remote = NULL;

	// We are only interested in resource actions
	if (action->rsc == NULL) {
	continue;
	}

	/* Special case: If we are clearing the failcount of an actual
	* remote connection resource, then make sure this happens before
	* any start of the resource in this transition.
	*/
	if (action->rsc->is_remote_node &&
	pcmk__str_eq(action->task, PCMK_ACTION_CLEAR_FAILCOUNT,
	pcmk__str_none)) {

	pcmk__new_ordering(action->rsc, NULL, action, action->rsc,
	pcmk__op_key(action->rsc->id, PCMK_ACTION_START,
	0),
	NULL, pcmk__ar_ordered, data_set);

	continue;
	}

	// We are only interested in actions assigned to a node
	if (action->node == NULL) {
	continue;
	}

	if (!pe__is_guest_or_remote_node(action->node)) {
	continue;
	}

	/* We are only interested in real actions.
	*
	* @TODO This is probably wrong; pseudo-actions might be converted to
	* real actions and vice versa later in update_actions() at the end of
	* pcmk__apply_orderings().
	*/
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	continue;
	}

	remote = action->node->details->remote_rsc;
	if (remote == NULL) {
	// Orphaned
	continue;
	}

	/* Another special case: if a resource is moving to a Pacemaker Remote
	* node, order the stop on the original node after any start of the
	* remote connection. This ensures that if the connection fails to
	* start, we leave the resource running on the original node.
	*/
	if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_none)) {
	for (GList *item = action->rsc->actions; item != NULL;
	item = item->next) {
	pe_action_t *rsc_action = item->data;

	if (!pe__same_node(rsc_action->node, action->node)
	&& pcmk__str_eq(rsc_action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	pcmk__new_ordering(remote, start_key(remote), NULL,
	action->rsc, NULL, rsc_action,
	pcmk__ar_ordered, data_set);
	}
	}
	}

	/* The action occurs across a remote connection, so create
	* ordering constraints that guarantee the action occurs while the node
	* is active (after start, before stop ... things like that).
	*
	* This is somewhat brittle in that we need to make sure the results of
	* this ordering are compatible with the result of get_router_node().
	* It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
	* of this logic rather than create_graph_action().
	*/
	if (remote->container) {
	crm_trace("Container ordering for %s", action->uuid);
	apply_container_ordering(action);

	} else {
	crm_trace("Remote ordering for %s", action->uuid);
	apply_remote_ordering(action);
	}
	}
	}

	/*!
	* \internal
	* \brief Check whether a node is a failed remote node
	*
	* \param[in] node Node to check
	*
	* \return true if \p node is a failed remote node, false otherwise
	*/
	bool
	pcmk__is_failed_remote_node(const pe_node_t *node)
	{
	return pe__is_remote_node(node) && (node->details->remote_rsc != NULL)
	&& (get_remote_node_state(node) == remote_state_failed);
	}

	/*!
	* \internal
	* \brief Check whether a given resource corresponds to a given node as guest
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is a guest node and \p rsc is its containing
	* resource, otherwise false
	*/
	bool
	pcmk__rsc_corresponds_to_guest(const pe_resource_t rsc, const pe_node_t node)
	{
	return (rsc != NULL) && (rsc->fillers != NULL) && (node != NULL)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container == rsc);
	}

	/*!
	* \internal
	* \brief Get proper connection host that a remote action must be routed through
	*
	* A remote connection resource might be starting, stopping, or migrating in the
	* same transition that an action needs to be executed on its Pacemaker Remote
	* node. Determine the proper node that the remote action should be routed
	* through.
	*
	* \param[in] action (Potentially remote) action to route
	*
	* \return Connection host that action should be routed through if remote,
	* otherwise NULL
	*/
	pe_node_t *
	pcmk__connection_host_for_action(const pe_action_t *action)
	{
	pe_node_t *began_on = NULL;
	pe_node_t *ended_on = NULL;
	bool partial_migration = false;
	const char *task = action->task;

	if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_none)
	\|\| !pe__is_guest_or_remote_node(action->node)) {
	return NULL;
	}

	CRM_ASSERT(action->node->details->remote_rsc != NULL);

	began_on = pe__current_node(action->node->details->remote_rsc);
	ended_on = action->node->details->remote_rsc->allocated_to;
	if (action->node->details->remote_rsc
	&& (action->node->details->remote_rsc->container == NULL)
	&& action->node->details->remote_rsc->partial_migration_target) {
	partial_migration = true;
	}

	if (began_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (starting)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	if (ended_on == NULL) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (stopping)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	if (pe__same_node(began_on, ended_on)) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (not moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* If we get here, the remote connection is moving during this transition.
	* This means some actions for resources behind the connection will get
	* routed through the cluster node the connection resource is currently on,
	* and others are routed through the cluster node the connection will end up
	* on.
	*/

	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	task = g_hash_table_lookup(action->meta, "notify_operation");
	}

	/*
	* Stop, demote, and migration actions must occur before the connection can
	* move (these actions are required before the remote resource can stop). In
	* this case, we know these actions have to be routed through the initial
	* cluster node the connection resource lived on before the move takes
	* place.
	*
	* The exception is a partial migration of a (non-guest) remote connection
	* resource; in that case, all actions (even these) will be ordered after
	* the connection's pseudo-start on the migration target, so the target is
	* the router node.
	*/
	if (pcmk__strcase_any_of(task, PCMK_ACTION_CANCEL, PCMK_ACTION_STOP,
	PCMK_ACTION_DEMOTE, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_MIGRATE_TO, NULL)
	&& !partial_migration) {
	crm_trace("Routing %s for %s through remote connection's "
	"current node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(began_on? began_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return began_on;
	}

	/* Everything else (start, promote, monitor, probe, refresh,
	* clear failcount, delete, ...) must occur after the connection starts on
	* the node it is moving to.
	*/
	crm_trace("Routing %s for %s through remote connection's "
	"next node %s (moving)%s",
	action->task, (action->rsc? action->rsc->id : "no resource"),
	(ended_on? ended_on->details->uname : "none"),
	partial_migration? " (partial migration)" : "");
	return ended_on;
	}

	/*!
	* \internal
	* \brief Replace remote connection's addr="#uname" with actual address
	*
	* REMOTE_CONTAINER_HACK: If a given resource is a remote connection resource
	* with its "addr" parameter set to "#uname", pull the actual value from the
	* parameters evaluated without a node (which was put there earlier in
	* pcmk__create_graph() when the bundle's expand() method was called).
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] params Resource parameters evaluated per node
	*/
	void
	pcmk__substitute_remote_addr(pe_resource_t rsc, GHashTable params)
	{
	const char *remote_addr = g_hash_table_lookup(params,
	XML_RSC_ATTR_REMOTE_RA_ADDR);

	if (pcmk__str_eq(remote_addr, "#uname", pcmk__str_none)) {
	GHashTable *base = pe_rsc_params(rsc, NULL, rsc->cluster);

	remote_addr = g_hash_table_lookup(base, XML_RSC_ATTR_REMOTE_RA_ADDR);
	if (remote_addr != NULL) {
	g_hash_table_insert(params, strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
	strdup(remote_addr));
	}
	}
	}

	/*!
	* \brief Add special bundle meta-attributes to XML
	*
	* If a given action will be executed on a guest node (including a bundle),
	* add the special bundle meta-attribute "container-attribute-target" and
	* environment variable "physical_host" as XML attributes (using meta-attribute
	* naming).
	*
	* \param[in,out] args_xml XML to add attributes to
	* \param[in] action Action to check
	*/
	void
	pcmk__add_bundle_meta_to_xml(xmlNode args_xml, const pe_action_t action)
	{
	const pe_node_t *guest = action->node;
	const pe_node_t *host = NULL;
	enum action_tasks task;

	if (!pe__is_guest_node(guest)) {
	return;
	}

	task = text2task(action->task);
	if ((task == pcmk_action_notify) \|\| (task == pcmk_action_notified)) {
	task = text2task(g_hash_table_lookup(action->meta, "notify_operation"));
	}

	switch (task) {
	case pcmk_action_stop:
	case pcmk_action_stopped:
	case pcmk_action_demote:
	case pcmk_action_demoted:
	// "Down" actions take place on guest's current host
	host = pe__current_node(guest->details->remote_rsc->container);
	break;

	case pcmk_action_start:
	case pcmk_action_started:
	case pcmk_action_monitor:
	case pcmk_action_promote:
	case pcmk_action_promoted:
	// "Up" actions take place on guest's next host
	host = guest->details->remote_rsc->container->allocated_to;
	break;

	default:
	break;
	}

	if (host != NULL) {
	hash2metafield((gpointer) XML_RSC_ATTR_TARGET,
	(gpointer) g_hash_table_lookup(action->rsc->meta,
	XML_RSC_ATTR_TARGET),
	(gpointer) args_xml);
	hash2metafield((gpointer) PCMK__ENV_PHYSICAL_HOST,
	(gpointer) host->details->uname,
	(gpointer) args_xml);
	}
	}
	diff --git a/lib/pengine/pe_notif.c b/lib/pengine/pe_notif.c
	index d20555fb49..692d91c4b2 100644
	--- a/lib/pengine/pe_notif.c
	+++ b/lib/pengine/pe_notif.c
	@@ -1,1005 +1,1006 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>

	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>

	#include "pe_status_private.h"

	typedef struct notify_entry_s {
	const pe_resource_t *rsc;
	const pe_node_t *node;
	} notify_entry_t;

	/*!
	* \internal
	* \brief Compare two notification entries
	*
	* Compare two notification entries, where the one with the alphabetically first
	* resource name (or if equal, node name) sorts as first, with NULL sorting as
	* less than non-NULL.
	*
	* \param[in] a First notification entry to compare
	* \param[in] b Second notification entry to compare
	*
	* \return -1 if \p a sorts before \p b, 0 if they are equal, otherwise 1
	*/
	static gint
	compare_notify_entries(gconstpointer a, gconstpointer b)
	{
	int tmp;
	const notify_entry_t *entry_a = a;
	const notify_entry_t *entry_b = b;

	// NULL a or b is not actually possible
	if ((entry_a == NULL) && (entry_b == NULL)) {
	return 0;
	}
	if (entry_a == NULL) {
	return 1;
	}
	if (entry_b == NULL) {
	return -1;
	}

	// NULL resources sort first
	if ((entry_a->rsc == NULL) && (entry_b->rsc == NULL)) {
	return 0;
	}
	if (entry_a->rsc == NULL) {
	return 1;
	}
	if (entry_b->rsc == NULL) {
	return -1;
	}

	// Compare resource names
	tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
	if (tmp != 0) {
	return tmp;
	}

	// Otherwise NULL nodes sort first
	if ((entry_a->node == NULL) && (entry_b->node == NULL)) {
	return 0;
	}
	if (entry_a->node == NULL) {
	return 1;
	}
	if (entry_b->node == NULL) {
	return -1;
	}

	// Finally, compare node names
	return strcmp(entry_a->node->details->id, entry_b->node->details->id);
	}

	/*!
	* \internal
	* \brief Duplicate a notification entry
	*
	* \param[in] entry Entry to duplicate
	*
	* \return Newly allocated duplicate of \p entry
	* \note It is the caller's responsibility to free the return value.
	*/
	static notify_entry_t *
	dup_notify_entry(const notify_entry_t *entry)
	{
	notify_entry_t *dup = calloc(1, sizeof(notify_entry_t));

	CRM_ASSERT(dup != NULL);
	dup->rsc = entry->rsc;
	dup->node = entry->node;
	return dup;
	}

	/*!
	* \internal
	* \brief Given a list of nodes, create strings with node names
	*
	* \param[in] list List of nodes (as pe_node_t *)
	* \param[out] all_node_names If not NULL, will be set to space-separated list
	* of the names of all nodes in \p list
	* \param[out] host_node_names Same as \p all_node_names, except active
	* guest nodes will list the name of their host
	*
	* \note The caller is responsible for freeing the output argument values using
	* \p g_string_free().
	*/
	static void
	get_node_names(const GList list, GString *all_node_names,
	GString **host_node_names)
	{
	if (all_node_names != NULL) {
	*all_node_names = NULL;
	}
	if (host_node_names != NULL) {
	*host_node_names = NULL;
	}

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	const pe_node_t node = (const pe_node_t ) iter->data;

	if (node->details->uname == NULL) {
	continue;
	}

	// Always add to list of all node names
	if (all_node_names != NULL) {
	pcmk__add_word(all_node_names, 1024, node->details->uname);
	}

	// Add to host node name list if appropriate
	if (host_node_names != NULL) {
	if (pe__is_guest_node(node)
	&& (node->details->remote_rsc->container->running_on != NULL)) {
	node = pe__current_node(node->details->remote_rsc->container);
	if (node->details->uname == NULL) {
	continue;
	}
	}
	pcmk__add_word(host_node_names, 1024, node->details->uname);
	}
	}

	if ((all_node_names != NULL) && (*all_node_names == NULL)) {
	*all_node_names = g_string_new(" ");
	}
	if ((host_node_names != NULL) && (*host_node_names == NULL)) {
	*host_node_names = g_string_new(" ");
	}
	}

	/*!
	* \internal
	* \brief Create strings of instance and node names from notification entries
	*
	* \param[in,out] list List of notification entries (will be sorted here)
	* \param[out] rsc_names If not NULL, will be set to space-separated list
	* of clone instances from \p list
	* \param[out] node_names If not NULL, will be set to space-separated list
	* of node names from \p list
	*
	* \return (Possibly new) head of sorted \p list
	* \note The caller is responsible for freeing the output argument values using
	* \p g_list_free_full() and \p g_string_free().
	*/
	static GList *
	notify_entries_to_strings(GList list, GString *rsc_names,
	GString **node_names)
	{
	const char *last_rsc_id = NULL;

	// Initialize output lists to NULL
	if (rsc_names != NULL) {
	*rsc_names = NULL;
	}
	if (node_names != NULL) {
	*node_names = NULL;
	}

	// Sort input list for user-friendliness (and ease of filtering duplicates)
	list = g_list_sort(list, compare_notify_entries);

	for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
	notify_entry_t entry = (notify_entry_t ) gIter->data;

	// Entry must have a resource (with ID)
	CRM_LOG_ASSERT((entry != NULL) && (entry->rsc != NULL)
	&& (entry->rsc->id != NULL));
	if ((entry == NULL) \|\| (entry->rsc == NULL)
	\|\| (entry->rsc->id == NULL)) {
	continue;
	}

	// Entry must have a node unless listing inactive resources
	CRM_LOG_ASSERT((node_names == NULL) \|\| (entry->node != NULL));
	if ((node_names != NULL) && (entry->node == NULL)) {
	continue;
	}

	// Don't add duplicates of a particular clone instance
	if (pcmk__str_eq(entry->rsc->id, last_rsc_id, pcmk__str_none)) {
	continue;
	}
	last_rsc_id = entry->rsc->id;

	if (rsc_names != NULL) {
	pcmk__add_word(rsc_names, 1024, entry->rsc->id);
	}
	if ((node_names != NULL) && (entry->node->details->uname != NULL)) {
	pcmk__add_word(node_names, 1024, entry->node->details->uname);
	}
	}

	// If there are no entries, return "empty" lists
	if ((rsc_names != NULL) && (*rsc_names == NULL)) {
	*rsc_names = g_string_new(" ");
	}
	if ((node_names != NULL) && (*node_names == NULL)) {
	*node_names = g_string_new(" ");
	}

	return list;
	}

	/*!
	* \internal
	* \brief Copy a meta-attribute into a notify action
	*
	* \param[in] key Name of meta-attribute to copy
	* \param[in] value Value of meta-attribute to copy
	* \param[in,out] user_data Notify action to copy into
	*/
	static void
	copy_meta_to_notify(gpointer key, gpointer value, gpointer user_data)
	{
	pe_action_t notify = (pe_action_t ) user_data;

	/* Any existing meta-attributes (for example, the action timeout) are for
	* the notify action itself, so don't override those.
	*/
	if (g_hash_table_lookup(notify->meta, (const char *) key) != NULL) {
	return;
	}

	g_hash_table_insert(notify->meta, strdup((const char *) key),
	strdup((const char *) value));
	}

	static void
	add_notify_data_to_action_meta(const notify_data_t n_data, pe_action_t action)
	{
	for (const GSList *item = n_data->keys; item; item = item->next) {
	const pcmk_nvpair_t nvpair = (const pcmk_nvpair_t ) item->data;

	add_hash_param(action->meta, nvpair->name, nvpair->value);
	}
	}

	/*!
	* \internal
	* \brief Create a new notify pseudo-action for a clone resource
	*
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in] action Action to use in notify action key
	* \param[in] notif_action PCMK_ACTION_NOTIFY or PCMK_ACTION_NOTIFIED
	* \param[in] notif_type "pre", "post", "confirmed-pre", "confirmed-post"
	*
	* \return Newly created notify pseudo-action
	*/
	static pe_action_t *
	new_notify_pseudo_action(pe_resource_t rsc, const pe_action_t action,
	const char notif_action, const char notif_type)
	{
	pe_action_t *notify = NULL;

	notify = custom_action(rsc,
	pcmk__notify_key(rsc->id, notif_type, action->task),
	notif_action, NULL,
	pcmk_is_set(action->flags, pcmk_action_optional),
	TRUE, rsc->cluster);
	pe__set_action_flags(notify, pcmk_action_pseudo);
	add_hash_param(notify->meta, "notify_key_type", notif_type);
	add_hash_param(notify->meta, "notify_key_operation", action->task);
	return notify;
	}

	/*!
	* \internal
	* \brief Create a new notify action for a clone instance
	*
	* \param[in,out] rsc Clone instance that notification is for
	* \param[in] node Node that notification is for
	* \param[in,out] op Action that notification is for
	* \param[in,out] notify_done Parent pseudo-action for notifications complete
	* \param[in] n_data Notification values to add to action meta-data
	*
	* \return Newly created notify action
	*/
	static pe_action_t *
	new_notify_action(pe_resource_t rsc, const pe_node_t node, pe_action_t *op,
	pe_action_t notify_done, const notify_data_t n_data)
	{
	char *key = NULL;
	pe_action_t *notify_action = NULL;
	const char *value = NULL;
	const char *task = NULL;
	const char *skip_reason = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL), return NULL);

	// Ensure we have all the info we need
	if (op == NULL) {
	skip_reason = "no action";
	} else if (notify_done == NULL) {
	skip_reason = "no parent notification";
	} else if (!node->details->online) {
	skip_reason = "node offline";
	} else if (!pcmk_is_set(op->flags, pcmk_action_runnable)) {
	skip_reason = "original action not runnable";
	}
	if (skip_reason != NULL) {
	pe_rsc_trace(rsc, "Skipping notify action for %s on %s: %s",
	rsc->id, pe__node_name(node), skip_reason);
	return NULL;
	}

	value = g_hash_table_lookup(op->meta, "notify_type"); // "pre" or "post"
	task = g_hash_table_lookup(op->meta, "notify_operation"); // original action

	pe_rsc_trace(rsc, "Creating notify action for %s on %s (%s-%s)",
	rsc->id, pe__node_name(node), value, task);

	// Create the notify action
	key = pcmk__notify_key(rsc->id, value, task);
	notify_action = custom_action(rsc, key, op->task, node,
	pcmk_is_set(op->flags, pcmk_action_optional),
	TRUE, rsc->cluster);

	// Add meta-data to notify action
	g_hash_table_foreach(op->meta, copy_meta_to_notify, notify_action);
	add_notify_data_to_action_meta(n_data, notify_action);

	// Order notify after original action and before parent notification
	order_actions(op, notify_action, pcmk__ar_ordered);
	order_actions(notify_action, notify_done, pcmk__ar_ordered);
	return notify_action;
	}

	/*!
	* \internal
	* \brief Create a new "post-" notify action for a clone instance
	*
	* \param[in,out] rsc Clone instance that notification is for
	* \param[in] node Node that notification is for
	* \param[in,out] n_data Notification values to add to action meta-data
	*/
	static void
	new_post_notify_action(pe_resource_t rsc, const pe_node_t node,
	notify_data_t *n_data)
	{
	pe_action_t *notify = NULL;

	CRM_ASSERT(n_data != NULL);

	// Create the "post-" notify action for specified instance
	notify = new_notify_action(rsc, node, n_data->post, n_data->post_done,
	n_data);
	if (notify != NULL) {
	notify->priority = INFINITY;
	}

	// Order recurring monitors after all "post-" notifications complete
	if (n_data->post_done == NULL) {
	return;
	}
	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pe_action_t mon = (pe_action_t ) iter->data;
	const char *interval_ms_s = NULL;

	interval_ms_s = g_hash_table_lookup(mon->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	\|\| pcmk__str_eq(mon->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	continue; // Not a recurring monitor
	}
	order_actions(n_data->post_done, mon, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Create and order notification pseudo-actions for a clone action
	*
	* In addition to the actual notify actions needed for each clone instance,
	* clone notifications also require pseudo-actions to provide ordering points
	* in the notification process. This creates the notification data, along with
	* appropriate pseudo-actions and their orderings.
	*
	* For example, the ordering sequence for starting a clone is:
	*
	* "pre-" notify pseudo-action for clone
	* -> "pre-" notify actions for each clone instance
	* -> "pre-" notifications complete pseudo-action for clone
	* -> start actions for each clone instance
	* -> "started" pseudo-action for clone
	* -> "post-" notify pseudo-action for clone
	* -> "post-" notify actions for each clone instance
	* -> "post-" notifications complete pseudo-action for clone
	*
	* \param[in,out] rsc Clone that notifications are for
	* \param[in] task Name of action that notifications are for
	* \param[in,out] action If not NULL, create a "pre-" pseudo-action ordered
	* before a "pre-" complete pseudo-action, ordered
	* before this action
	* \param[in,out] complete If not NULL, create a "post-" pseudo-action ordered
	* after this action, and a "post-" complete
	* pseudo-action ordered after that
	*
	* \return Newly created notification data
	*/
	notify_data_t *
	pe__action_notif_pseudo_ops(pe_resource_t rsc, const char task,
	pe_action_t action, pe_action_t complete)
	{
	notify_data_t *n_data = NULL;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_notify)) {
	return NULL;
	}

	n_data = calloc(1, sizeof(notify_data_t));
	CRM_ASSERT(n_data != NULL);

	n_data->action = task;

	if (action != NULL) { // Need "pre-" pseudo-actions

	// Create "pre-" notify pseudo-action for clone
	n_data->pre = new_notify_pseudo_action(rsc, action, PCMK_ACTION_NOTIFY,
	"pre");
	pe__set_action_flags(n_data->pre, pcmk_action_runnable);
	add_hash_param(n_data->pre->meta, "notify_type", "pre");
	add_hash_param(n_data->pre->meta, "notify_operation", n_data->action);

	// Create "pre-" notifications complete pseudo-action for clone
	n_data->pre_done = new_notify_pseudo_action(rsc, action,
	PCMK_ACTION_NOTIFIED,
	"confirmed-pre");
	pe__set_action_flags(n_data->pre_done, pcmk_action_runnable);
	add_hash_param(n_data->pre_done->meta, "notify_type", "pre");
	add_hash_param(n_data->pre_done->meta,
	"notify_operation", n_data->action);

	// Order "pre-" -> "pre-" complete -> original action
	order_actions(n_data->pre, n_data->pre_done, pcmk__ar_ordered);
	order_actions(n_data->pre_done, action, pcmk__ar_ordered);
	}

	if (complete != NULL) { // Need "post-" pseudo-actions

	// Create "post-" notify pseudo-action for clone
	n_data->post = new_notify_pseudo_action(rsc, complete,
	PCMK_ACTION_NOTIFY, "post");
	n_data->post->priority = INFINITY;
	if (pcmk_is_set(complete->flags, pcmk_action_runnable)) {
	pe__set_action_flags(n_data->post, pcmk_action_runnable);
	} else {
	pe__clear_action_flags(n_data->post, pcmk_action_runnable);
	}
	add_hash_param(n_data->post->meta, "notify_type", "post");
	add_hash_param(n_data->post->meta, "notify_operation", n_data->action);

	// Create "post-" notifications complete pseudo-action for clone
	n_data->post_done = new_notify_pseudo_action(rsc, complete,
	PCMK_ACTION_NOTIFIED,
	"confirmed-post");
	n_data->post_done->priority = INFINITY;
	if (pcmk_is_set(complete->flags, pcmk_action_runnable)) {
	pe__set_action_flags(n_data->post_done, pcmk_action_runnable);
	} else {
	pe__clear_action_flags(n_data->post_done, pcmk_action_runnable);
	}
	add_hash_param(n_data->post_done->meta, "notify_type", "post");
	add_hash_param(n_data->post_done->meta,
	"notify_operation", n_data->action);

	// Order original action complete -> "post-" -> "post-" complete
	- order_actions(complete, n_data->post, pe_order_implies_then);
	- order_actions(n_data->post, n_data->post_done, pe_order_implies_then);
	+ order_actions(complete, n_data->post, pcmk__ar_first_implies_then);
	+ order_actions(n_data->post, n_data->post_done,
	+ pcmk__ar_first_implies_then);
	}

	// If we created both, order "pre-" complete -> "post-"
	if ((action != NULL) && (complete != NULL)) {
	order_actions(n_data->pre_done, n_data->post, pcmk__ar_ordered);
	}
	return n_data;
	}

	/*!
	* \internal
	* \brief Create a new notification entry
	*
	* \param[in] rsc Resource for notification
	* \param[in] node Node for notification
	*
	* \return Newly allocated notification entry
	* \note The caller is responsible for freeing the return value.
	*/
	static notify_entry_t *
	new_notify_entry(const pe_resource_t rsc, const pe_node_t node)
	{
	notify_entry_t *entry = calloc(1, sizeof(notify_entry_t));

	CRM_ASSERT(entry != NULL);
	entry->rsc = rsc;
	entry->node = node;
	return entry;
	}

	/*!
	* \internal
	* \brief Add notification data for resource state and optionally actions
	*
	* \param[in] rsc Clone or clone instance being notified
	* \param[in] activity Whether to add notification entries for actions
	* \param[in,out] n_data Notification data for clone
	*/
	static void
	collect_resource_data(const pe_resource_t *rsc, bool activity,
	notify_data_t *n_data)
	{
	const GList *iter = NULL;
	notify_entry_t *entry = NULL;
	const pe_node_t *node = NULL;

	if (n_data == NULL) {
	return;
	}

	if (n_data->allowed_nodes == NULL) {
	n_data->allowed_nodes = rsc->allowed_nodes;
	}

	// If this is a clone, call recursively for each instance
	if (rsc->children != NULL) {
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	const pe_resource_t child = (const pe_resource_t ) iter->data;

	collect_resource_data(child, activity, n_data);
	}
	return;
	}

	// This is a notification for a single clone instance

	if (rsc->running_on != NULL) {
	node = rsc->running_on->data; // First is sufficient
	}
	entry = new_notify_entry(rsc, node);

	// Add notification indicating the resource state
	switch (rsc->role) {
	case pcmk_role_stopped:
	n_data->inactive = g_list_prepend(n_data->inactive, entry);
	break;

	case pcmk_role_started:
	n_data->active = g_list_prepend(n_data->active, entry);
	break;

	case pcmk_role_unpromoted:
	n_data->unpromoted = g_list_prepend(n_data->unpromoted, entry);
	n_data->active = g_list_prepend(n_data->active,
	dup_notify_entry(entry));
	break;

	case pcmk_role_promoted:
	n_data->promoted = g_list_prepend(n_data->promoted, entry);
	n_data->active = g_list_prepend(n_data->active,
	dup_notify_entry(entry));
	break;

	default:
	crm_err("Resource %s role on %s (%s) is not supported for "
	"notifications (bug?)",
	rsc->id, pe__node_name(node), role2text(rsc->role));
	free(entry);
	break;
	}

	if (!activity) {
	return;
	}

	// Add notification entries for each of the resource's actions
	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	const pe_action_t op = (const pe_action_t ) iter->data;

	if (!pcmk_is_set(op->flags, pcmk_action_optional)
	&& (op->node != NULL)) {
	enum action_tasks task = text2task(op->task);

	if ((task == pcmk_action_stop) && op->node->details->unclean) {
	// Create anyway (additional noise if node can't be fenced)
	} else if (!pcmk_is_set(op->flags, pcmk_action_runnable)) {
	continue;
	}

	entry = new_notify_entry(rsc, op->node);

	switch (task) {
	case pcmk_action_start:
	n_data->start = g_list_prepend(n_data->start, entry);
	break;
	case pcmk_action_stop:
	n_data->stop = g_list_prepend(n_data->stop, entry);
	break;
	case pcmk_action_promote:
	n_data->promote = g_list_prepend(n_data->promote, entry);
	break;
	case pcmk_action_demote:
	n_data->demote = g_list_prepend(n_data->demote, entry);
	break;
	default:
	free(entry);
	break;
	}
	}
	}
	}

	// For (char *) value
	#define add_notify_env(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, value); \
	} while (0)

	// For (GString *) value
	#define add_notify_env_gs(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, \
	(const char *) value->str); \
	} while (0)

	// For (GString *) value
	#define add_notify_env_free_gs(n_data, key, value) do { \
	n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, \
	(const char *) value->str); \
	g_string_free(value, TRUE); value = NULL; \
	} while (0)

	/*!
	* \internal
	* \brief Create notification name/value pairs from structured data
	*
	* \param[in] rsc Resource that notification is for
	* \param[in,out] n_data Notification data
	*/
	static void
	add_notif_keys(const pe_resource_t rsc, notify_data_t n_data)
	{
	bool required = false; // Whether to make notify actions required
	GString *rsc_list = NULL;
	GString *node_list = NULL;
	GString *metal_list = NULL;
	const char *source = NULL;
	GList *nodes = NULL;

	n_data->stop = notify_entries_to_strings(n_data->stop,
	&rsc_list, &node_list);
	if ((strcmp(" ", (const char *) rsc_list->str) != 0)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_STOP, pcmk__str_none)) {
	required = true;
	}
	add_notify_env_free_gs(n_data, "notify_stop_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_stop_uname", node_list);

	if ((n_data->start != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_START, pcmk__str_none)) {
	required = true;
	}
	n_data->start = notify_entries_to_strings(n_data->start,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_start_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_start_uname", node_list);

	if ((n_data->demote != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	required = true;
	}
	n_data->demote = notify_entries_to_strings(n_data->demote,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_demote_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_demote_uname", node_list);

	if ((n_data->promote != NULL)
	&& pcmk__str_eq(n_data->action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	required = true;
	}
	n_data->promote = notify_entries_to_strings(n_data->promote,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_promote_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_promote_uname", node_list);

	n_data->active = notify_entries_to_strings(n_data->active,
	&rsc_list, &node_list);
	add_notify_env_free_gs(n_data, "notify_active_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_active_uname", node_list);

	n_data->unpromoted = notify_entries_to_strings(n_data->unpromoted,
	&rsc_list, &node_list);
	add_notify_env_gs(n_data, "notify_unpromoted_resource", rsc_list);
	add_notify_env_gs(n_data, "notify_unpromoted_uname", node_list);

	// Deprecated: kept for backward compatibility with older resource agents
	add_notify_env_free_gs(n_data, "notify_slave_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_slave_uname", node_list);

	n_data->promoted = notify_entries_to_strings(n_data->promoted,
	&rsc_list, &node_list);
	add_notify_env_gs(n_data, "notify_promoted_resource", rsc_list);
	add_notify_env_gs(n_data, "notify_promoted_uname", node_list);

	// Deprecated: kept for backward compatibility with older resource agents
	add_notify_env_free_gs(n_data, "notify_master_resource", rsc_list);
	add_notify_env_free_gs(n_data, "notify_master_uname", node_list);

	n_data->inactive = notify_entries_to_strings(n_data->inactive,
	&rsc_list, NULL);
	add_notify_env_free_gs(n_data, "notify_inactive_resource", rsc_list);

	nodes = g_hash_table_get_values(n_data->allowed_nodes);
	if (!pcmk__is_daemon) {
	/* For display purposes, sort the node list, for consistent
	* regression test output (while avoiding the performance hit
	* for the live cluster).
	*/
	nodes = g_list_sort(nodes, pe__cmp_node_name);
	}
	get_node_names(nodes, &node_list, NULL);
	add_notify_env_free_gs(n_data, "notify_available_uname", node_list);
	g_list_free(nodes);

	source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET);
	if (pcmk__str_eq("host", source, pcmk__str_none)) {
	get_node_names(rsc->cluster->nodes, &node_list, &metal_list);
	add_notify_env_free_gs(n_data, "notify_all_hosts", metal_list);
	} else {
	get_node_names(rsc->cluster->nodes, &node_list, NULL);
	}
	add_notify_env_free_gs(n_data, "notify_all_uname", node_list);

	if (required && (n_data->pre != NULL)) {
	pe__clear_action_flags(n_data->pre, pcmk_action_optional);
	pe__clear_action_flags(n_data->pre_done, pcmk_action_optional);
	}

	if (required && (n_data->post != NULL)) {
	pe__clear_action_flags(n_data->post, pcmk_action_optional);
	pe__clear_action_flags(n_data->post_done, pcmk_action_optional);
	}
	}

	/*
	* \internal
	* \brief Find any remote connection start relevant to an action
	*
	* \param[in] action Action to check
	*
	* \return If action is behind a remote connection, connection's start
	*/
	static pe_action_t *
	find_remote_start(pe_action_t *action)
	{
	if ((action != NULL) && (action->node != NULL)) {
	pe_resource_t *remote_rsc = action->node->details->remote_rsc;

	if (remote_rsc != NULL) {
	return find_first_action(remote_rsc->actions, NULL,
	PCMK_ACTION_START,
	NULL);
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Create notify actions, and add notify data to original actions
	*
	* \param[in,out] rsc Clone or clone instance that notification is for
	* \param[in,out] n_data Clone notification data for some action
	*/
	static void
	create_notify_actions(pe_resource_t rsc, notify_data_t n_data)
	{
	GList *iter = NULL;
	pe_action_t *stop = NULL;
	pe_action_t *start = NULL;
	enum action_tasks task = text2task(n_data->action);

	// If this is a clone, call recursively for each instance
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, (GFunc) create_notify_actions, n_data);
	return;
	}

	// Add notification meta-attributes to original actions
	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	pe_action_t op = (pe_action_t ) iter->data;

	if (!pcmk_is_set(op->flags, pcmk_action_optional)
	&& (op->node != NULL)) {
	switch (text2task(op->task)) {
	case pcmk_action_start:
	case pcmk_action_stop:
	case pcmk_action_promote:
	case pcmk_action_demote:
	add_notify_data_to_action_meta(n_data, op);
	break;
	default:
	break;
	}
	}
	}

	// Skip notify action itself if original action was not needed
	switch (task) {
	case pcmk_action_start:
	if (n_data->start == NULL) {
	pe_rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	case pcmk_action_promote:
	if (n_data->promote == NULL) {
	pe_rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	case pcmk_action_demote:
	if (n_data->demote == NULL) {
	pe_rsc_trace(rsc, "No notify action needed for %s %s",
	rsc->id, n_data->action);
	return;
	}
	break;

	default:
	// We cannot do same for stop because it might be implied by fencing
	break;
	}

	pe_rsc_trace(rsc, "Creating notify actions for %s %s",
	rsc->id, n_data->action);

	// Create notify actions for stop or demote
	if ((rsc->role != pcmk_role_stopped)
	&& ((task == pcmk_action_stop) \|\| (task == pcmk_action_demote))) {

	stop = find_first_action(rsc->actions, NULL, PCMK_ACTION_STOP, NULL);

	for (iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t current_node = (pe_node_t ) iter->data;

	/* If a stop is a pseudo-action implied by fencing, don't try to
	* notify the node getting fenced.
	*/
	if ((stop != NULL)
	&& pcmk_is_set(stop->flags, pcmk_action_pseudo)
	&& (current_node->details->unclean
	\|\| current_node->details->remote_requires_reset)) {
	continue;
	}

	new_notify_action(rsc, current_node, n_data->pre,
	n_data->pre_done, n_data);

	if ((task == pcmk_action_demote) \|\| (stop == NULL)
	\|\| pcmk_is_set(stop->flags, pcmk_action_optional)) {
	new_post_notify_action(rsc, current_node, n_data);
	}
	}
	}

	// Create notify actions for start or promote
	if ((rsc->next_role != pcmk_role_stopped)
	&& ((task == pcmk_action_start) \|\| (task == pcmk_action_promote))) {

	start = find_first_action(rsc->actions, NULL, PCMK_ACTION_START, NULL);
	if (start != NULL) {
	pe_action_t *remote_start = find_remote_start(start);

	if ((remote_start != NULL)
	&& !pcmk_is_set(remote_start->flags, pcmk_action_runnable)) {
	/* Start and promote actions for a clone instance behind
	* a Pacemaker Remote connection happen after the
	* connection starts. If the connection start is blocked, do
	* not schedule notifications for these actions.
	*/
	return;
	}
	}
	if (rsc->allocated_to == NULL) {
	pe_proc_err("Next role '%s' but %s is not allocated",
	role2text(rsc->next_role), rsc->id);
	return;
	}
	if ((task != pcmk_action_start) \|\| (start == NULL)
	\|\| pcmk_is_set(start->flags, pcmk_action_optional)) {

	new_notify_action(rsc, rsc->allocated_to, n_data->pre,
	n_data->pre_done, n_data);
	}
	new_post_notify_action(rsc, rsc->allocated_to, n_data);
	}
	}

	/*!
	* \internal
	* \brief Create notification data and actions for one clone action
	*
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in,out] n_data Clone notification data for some action
	*/
	void
	pe__create_action_notifications(pe_resource_t rsc, notify_data_t n_data)
	{
	if ((rsc == NULL) \|\| (n_data == NULL)) {
	return;
	}
	collect_resource_data(rsc, true, n_data);
	add_notif_keys(rsc, n_data);
	create_notify_actions(rsc, n_data);
	}

	/*!
	* \internal
	* \brief Free notification data for one action
	*
	* \param[in,out] n_data Notification data to free
	*/
	void
	pe__free_action_notification_data(notify_data_t *n_data)
	{
	if (n_data == NULL) {
	return;
	}
	g_list_free_full(n_data->stop, free);
	g_list_free_full(n_data->start, free);
	g_list_free_full(n_data->demote, free);
	g_list_free_full(n_data->promote, free);
	g_list_free_full(n_data->promoted, free);
	g_list_free_full(n_data->unpromoted, free);
	g_list_free_full(n_data->active, free);
	g_list_free_full(n_data->inactive, free);
	pcmk_free_nvpairs(n_data->keys);
	free(n_data);
	}

	/*!
	* \internal
	* \brief Order clone "notifications complete" pseudo-action after fencing
	*
	* If a stop action is implied by fencing, the usual notification pseudo-actions
	* will not be sufficient to order things properly, or even create all needed
	* notifications if the clone is also stopping on another node, and another
	* clone is ordered after it. This function creates new notification
	* pseudo-actions relative to the fencing to ensure everything works properly.
	*
	* \param[in] stop Stop action implied by fencing
	* \param[in,out] rsc Clone resource that notification is for
	* \param[in,out] stonith_op Fencing action that implies \p stop
	*/
	void
	pe__order_notifs_after_fencing(const pe_action_t stop, pe_resource_t rsc,
	pe_action_t *stonith_op)
	{
	notify_data_t *n_data;

	crm_info("Ordering notifications for implied %s after fencing", stop->uuid);
	n_data = pe__action_notif_pseudo_ops(rsc, PCMK_ACTION_STOP, NULL,
	stonith_op);

	if (n_data != NULL) {
	collect_resource_data(rsc, false, n_data);
	add_notify_env(n_data, "notify_stop_resource", rsc->id);
	add_notify_env(n_data, "notify_stop_uname", stop->node->details->uname);
	create_notify_actions(uber_parent(rsc), n_data);
	pe__free_action_notification_data(n_data);
	}
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 2f980f1a65..0b57fef9aa 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,4963 +1,4963 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pe_resource_t *rsc; // Resource that history is for
	pe_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(data_set, option, flag) do { \
	const char *scf_value = pe_pref((data_set)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(data_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", \
	crm_system_name, (data_set)->flags, \
	(flag), #flag); \
	} else { \
	(data_set)->flags = pcmk__clear_flags_as(__func__, __LINE__,\
	LOG_TRACE, "Working set", \
	crm_system_name, (data_set)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	xmlNode **last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pe_working_set_t *data_set,
	pe_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pe_node_t node,
	bool overwrite, pe_working_set_t *data_set);
	static void determine_online_status(const xmlNode *node_state,
	pe_node_t *this_node,
	pe_working_set_t *data_set);

	static void unpack_node_lrm(pe_node_t node, const xmlNode xml,
	pe_working_set_t *data_set);


	// Bitmask for warnings we only want to print once
	uint32_t pe_wo = 0;

	static gboolean
	is_dangling_guest_node(pe_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] data_set Current working set of cluster
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider `priority-fencing-delay`
	*/
	void
	pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pe_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pe__node_name(node), reason, rsc->id);
	} else {
	crm_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pe__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pe__node_name(node), reason);
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pe_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pe__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	crm_warn("Remote node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply `priority-fencing-delay` for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pe__node_name(node),
	pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	crm_warn("Cluster node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
	"and @" XML_NVPAIR_ATTR_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pe_working_set_t data_set)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(data_set->flags, flag)) {
	result = xpath_search(data_set->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pe__set_working_set_flags(data_set, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode * config, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	data_set->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set);

	verify_pe_options(data_set->config_hash);

	set_config_flag(data_set, "enable-startup-probes",
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(data_set->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and stonith-watchdog-timeout is nonzero");
	pe__set_working_set_flags(data_set, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pcmk_sched_fencing_enabled);
	if (pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	if (!strcmp(data_set->stonith_action, "poweroff")) {
	pe_warn_once(pe_wo_poweroff,
	"Support for stonith-action of 'poweroff' is deprecated "
	"and will be removed in a future release (use 'off' instead)");
	data_set->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "concurrent-fencing",
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(data_set->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(data_set->config_hash,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
	if (value) {
	data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
	crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
	}

	set_config_flag(data_set, "stop-all-resources", pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(data_set->flags, pcmk_sched_stop_all)));

	set_config_flag(data_set, "symmetric-cluster",
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(data_set->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	data_set->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	data_set->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	data_set->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(data_set->flags, pcmk_sched_quorate)) {
	data_set->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
	data_set->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
	"fencing is disabled");
	data_set->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources",
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(data_set->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(data_set, "stop-orphan-actions",
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(data_set->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(data_set->config_hash, "remove-after-stop");
	if (value != NULL) {
	if (crm_is_true(value)) {
	pe__set_working_set_flags(data_set, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pe_warn_once(pe_wo_remove_after,
	"Support for the remove-after-stop cluster property is"
	" deprecated and will be removed in a future release");
	#endif
	} else {
	pe__clear_working_set_flags(data_set, pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(data_set, "maintenance-mode", pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(data_set->flags,
	pcmk_sched_in_maintenance)));

	set_config_flag(data_set, "start-failure-is-fatal",
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(data_set->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(data_set, "startup-fencing",
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(data_set->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(data_set);

	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
	crm_trace("Placement strategy: %s", data_set->placement_strategy);

	set_config_flag(data_set, "shutdown-lock", pcmk_sched_shutdown_lock);
	if (pcmk_is_set(data_set->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(data_set->config_hash,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
	data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(data_set->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(data_set->config_hash,
	XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT);
	data_set->node_pending_timeout = crm_parse_interval_spec(value) / 1000;
	crm_trace("Node pending timeout is %us", data_set->node_pending_timeout);

	return TRUE;
	}

	pe_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pe_working_set_t data_set)
	{
	pe_node_t *new_node = NULL;

	if (pe_find_node(data_set->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pe_node_t));
	if (new_node == NULL) {
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = data_set;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	pe__set_working_set_flags(data_set, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pe_warn_once(pe_wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pe_working_set_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (pcmk__str_eq(name, XML_RSC_ATTR_REMOTE_NODE, pcmk__str_casei)) {
	remote_name = value;
	} else if (pcmk__str_eq(name, "remote-addr", pcmk__str_casei)) {
	remote_server = value;
	} else if (pcmk__str_eq(name, "remote-port", pcmk__str_casei)) {
	remote_port = value;
	} else if (pcmk__str_eq(name, "remote-connect-timeout", pcmk__str_casei)) {
	connect_timeout = value;
	} else if (pcmk__str_eq(name, "remote-allow-migrate", pcmk__str_casei)) {
	remote_allow_migrate=value;
	} else if (pcmk__str_eq(name, XML_RSC_ATTR_MANAGED, pcmk__str_casei)) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pe_working_set_t data_set, pe_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(data_set->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	pe_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, data_set);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(data_set, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, data_set);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
	crm_info("Creating a fake local node");
	pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
	data_set);
	}

	return TRUE;
	}

	static void
	setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, data_set);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pe_resource_t *container = pe_find_resource(data_set->resources, container_id);

	if (container) {
	rsc->container = container;
	pe__set_resource_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
	} else {
	pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	data_set);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pe_working_set_t data_set, pe_resource_t new_rsc)
	{
	pe_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(data_set->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(data_set->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pe_rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pe__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(data_set, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pe_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] data_set Where to put resource information
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pe_working_set_t data_set)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	data_set->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pe_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(data_set->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(data_set->template_rsc_sets, strdup(id), NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " XML_ATTR_ID "='%s'>",
	xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	data_set) == pcmk_rc_ok) {
	data_set->resources = g_list_append(data_set->resources, new_rsc);
	pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	setup_container(rsc, data_set);
	link_rsc2remotenode(data_set, rsc);
	}

	data_set->resources = g_list_sort(data_set->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(data_set->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(data_set->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
	{
	xmlNode *xml_tag = NULL;

	data_set->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pe_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, XML_ATTR_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, data_set);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pe_node_t this_node, const xmlNode state,
	pe_working_set_t *data_set)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pe_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pe__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pe__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pe__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pe__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] data_set Cluster working set
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pe_node_t node,
	pe_working_set_t *data_set)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, data_set);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pe__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute for %s because disabling resource discovery "
	"is not allowed for cluster nodes", pe__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] data_set Cluster working set
	*/
	static void
	unpack_node_state(const xmlNode state, pe_working_set_t data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pe_node_t *this_node = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	if (id == NULL) {
	crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
	XML_ATTR_ID);
	return;
	}

	uname = crm_element_value(state, XML_ATTR_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created node_state entry doesn't have an uname
	* yet. We should recognize the node as `pending` and wait for it to
	* join CPG.
	*/
	crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	XML_ATTR_UNAME, id);
	}

	this_node = pe_find_node_any(data_set->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, data_set);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pe__node_name(this_node), id);
	determine_online_status(state, this_node, data_set);

	if (!pcmk_is_set(data_set->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (data_set->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(data_set, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pe_working_set_t *data_set)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, XML_ATTR_UNAME);
	pe_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(data_set->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pe_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pe_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(data_set->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(data_set->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);
	unpack_handle_remote_attrs(this_node, state, data_set);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, data_set);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t * data_set)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (data_set->tickets == NULL) {
	data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, data_set);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, data_set);
	}
	}

	while (unpack_node_history(status, FALSE, data_set) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(data_set->flags,
	pcmk_sched_fencing_enabled),
	data_set);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (data_set->stop_needed != NULL) {
	for (GList *item = data_set->stop_needed; item; item = item->next) {
	pe_resource_t *container = item->data;
	pe_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(data_set->stop_needed);
	data_set->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(data_set, this_node);
	}
	}

	return TRUE;
	}

	static gboolean
	determine_online_status_no_fencing(pe_working_set_t *data_set,
	const xmlNode *node_state,
	pe_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	int member = false;
	bool crmd_online = false;
	long long when_member = 0;
	long long when_online = 0;

	// @COMPAT DCs < 2.1.7 use boolean instead of time for cluster membership
	if (crm_str_to_boolean(in_cluster, &member) != 1) {
	pcmk__scan_ll(in_cluster, &when_member, 0LL);
	member = (when_member > 0) ? true : false;
	}

	if (pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei)) {
	crmd_online = true;

	} else if (pcmk__str_eq(is_peer, OFFLINESTATUS, pcmk__str_casei)) {
	crmd_online = false;

	} else {
	pcmk__scan_ll(is_peer, &when_online, 0LL);
	crmd_online = (when_online > 0) ? true : false;
	}

	if (!member) {
	crm_trace("Node is down: in_cluster=%s",
	pcmk__s(in_cluster, "<null>"));

	} else if (crmd_online) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Controller is down: "
	"in_cluster=%s is_peer=%s join=%s expected=%s",
	pcmk__s(in_cluster, "<null>"), pcmk__s(is_peer, "<null>"),
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
	crm_info("in_cluster=%s is_peer=%s join=%s expected=%s",
	pcmk__s(in_cluster, "<null>"), pcmk__s(is_peer, "<null>"),
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(pe_working_set_t *data_set,
	const xmlNode node_state, pe_node_t this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	bool crmd_online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *terminate = pe_node_attribute_raw(this_node, "terminate");
	int member = false;
	long long when_member = 0;
	long long when_online = 0;

	/*
	- XML_NODE_JOIN_STATE ::= member\|down\|pending\|banned
	- XML_NODE_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- XML_NODE_IN_CLUSTER ::= true\|false
	- XML_NODE_IS_PEER ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- XML_NODE_IN_CLUSTER ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- XML_NODE_IS_PEER ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	if (crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if (terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];

	if (t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	crm_trace("%s: in_cluster=%s is_peer=%s join=%s expected=%s term=%d",
	pe__node_name(this_node), pcmk__s(in_cluster, "<null>"),
	pcmk__s(is_peer, "<null>"), pcmk__s(join, "<null>"),
	pcmk__s(exp_state, "<null>"), do_terminate);

	/* @COMPAT with boolean values of XML_NODE_IN_CLUSTER recorded for
	* DCs < 2.1.7
	*/
	if (crm_str_to_boolean(in_cluster, &member) != 1) {
	pcmk__scan_ll(in_cluster, &when_member, 0LL);
	member = (when_member > 0) ? true : false;
	}

	online = member;

	/* @COMPAT with "online"/"offline" values of XML_NODE_IS_PEER recorded for
	* DCs < 2.1.7
	*/
	if (pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei)) {
	crmd_online = true;

	} else if (pcmk__str_eq(is_peer, OFFLINESTATUS, pcmk__str_casei)) {
	crmd_online = false;

	} else {
	pcmk__scan_ll(is_peer, &when_online, 0LL);
	crmd_online = (when_online > 0) ? true : false;
	}

	if (exp_state == NULL) {
	exp_state = CRMD_JOINSTATE_DOWN;
	}

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pe__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	online = crmd_online;

	} else if (in_cluster == NULL) {
	pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_casei)) {
	pe_fence_node(data_set, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (do_terminate == FALSE && pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN, pcmk__str_casei)) {

	if (when_member > 0
	&& when_online == 0
	&& (get_effective_time(data_set) - when_member
	>= data_set->node_pending_timeout)) {
	pe_fence_node(data_set, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if (member \|\| crmd_online) {
	crm_info("- %s is not ready to run resources",
	pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pe__node_name(this_node));
	}

	} else if (do_terminate && pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_casei)
	&& !member && !crmd_online) {
	crm_info("%s was just shot", pe__node_name(this_node));
	online = FALSE;

	} else if (!member) {
	// Consider `priority-fencing-delay` for lost nodes
	pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);

	} else if (!crmd_online) {
	pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */
	} else if (do_terminate) {
	pe_fence_node(data_set, this_node, "termination was requested", FALSE);

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	crm_info("%s is active", pe__node_name(this_node));

	} else if (pcmk__strcase_any_of(join, CRMD_JOINSTATE_PENDING, CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
	crm_warn("%s: in-cluster=%s is-peer=%s join=%s expected=%s term=%d shutdown=%d",
	pe__node_name(this_node), pcmk__s(in_cluster, "<null>"),
	pcmk__s(is_peer, "<null>"), pcmk__s(join, "<null>"),
	pcmk__s(exp_state, "<null>"), do_terminate,
	this_node->details->shutdown);
	}

	return online;
	}

	static void
	determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
	{
	pe_resource_t *rsc = this_node->details->remote_rsc;
	pe_resource_t *container = NULL;
	pe_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pe_node_t this_node,
	pe_working_set_t *data_set)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(data_set, node_state, this_node);

	} else {
	online = determine_online_status_fencing(data_set, node_state, this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pe__node_name(this_node));

	} else if (this_node->details->unclean) {
	pe_proc_warn("%s is unclean", pe__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pe__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pe__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pe_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pe_working_set_t *data_set)
	{
	pe_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, data_set) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pe_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(data_set->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
	}
	link_rsc2remotenode(data_set, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pe__set_resource_flags(rsc, pcmk_rsc_removed_filler);
	}
	pe__set_resource_flags(rsc, pcmk_rsc_removed);
	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] data_set Cluster working set
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pe_resource_t *
	create_anonymous_orphan(pe_resource_t parent, const char rsc_id,
	const pe_node_t node, pe_working_set_t data_set)
	{
	pe_resource_t *top = pe__create_clone_child(parent, data_set);

	// find_rsc() because we might be a cloned group
	pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pe__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of clone-max instances);
	* (3) a newly created orphan (i.e. clone-max instances are already active).
	*
	* \param[in,out] data_set Cluster information
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (without instance)
	*/
	static pe_resource_t *
	find_anonymous_clone(pe_working_set_t data_set, const pe_node_t node,
	pe_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pe_resource_t *rsc = NULL;
	pe_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pe_rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pe__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pe_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if globally-unique
	* was flipped from true to false); and
	* (3) when we re-run calculations on the same data set as part of a
	* simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pe_node_t *)locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if globally-unique is switched from true to
	* false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pe__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pe_rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, data_set)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
	pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pe_resource_t *
	unpack_find_resource(pe_working_set_t data_set, const pe_node_t node,
	const char *rsc_id)
	{
	pe_resource_t *rsc = NULL;
	pe_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when clone-max=0, we create
	* a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(data_set, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pe__node_name(node), rsc->id,
	(pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : ""));
	}
	return rsc;
	}

	static pe_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pe_node_t node,
	pe_working_set_t *data_set)
	{
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(data_set->flags, pcmk_sched_stop_removed_resources)) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
	enum action_fail_response on_fail)
	{
	pe_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pe__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pe_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pe_node_t *n = pe__copy_node(node);

	pe_rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pe__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pe__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t stop = (pe_action_t ) gIter->data;

	pe__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pe_node_t * node, pe_resource_t * rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pe_working_set_t data_set)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pe__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pe_rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pe__node_name(node));
	continue;

	} else if (counter < start_index) {
	pe_rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pe__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pe_rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pe__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pe_rsc_trace(rsc, "Skipping %s on %s: status",
	id, pe__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pe_rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pe_resource_t rsc,
	const pe_node_t node, pe_working_set_t data_set)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((data_set->shutdown_lock > 0)
	&& (get_effective_time(data_set)
	> (lock_time + data_set->shutdown_lock))) {
	pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pe__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pe_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] data_set Cluster working set
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pe_resource_t *
	unpack_lrm_resource(pe_node_t node, const xmlNode lrm_resource,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pe_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	crm_warn("Ignoring malformed " XML_LRM_TAG_RESOURCE
	" entry without id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pe__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(data_set->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, data_set);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(data_set->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, data_set);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, XML_RSC_ATTR_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pe_working_set_t *data_set)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pe_resource_t *rsc;
	pe_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(data_set->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(data_set->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] data_set Cluster working set
	*/
	static void
	unpack_node_lrm(pe_node_t node, const xmlNode xml, pe_working_set_t *data_set)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pe_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, data_set);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, data_set);
	}
	}

	static void
	set_active(pe_resource_t * rsc)
	{
	const pe_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pe_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pe_working_set_t *data_set)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, data_set->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pe_working_set_t *data_set)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, data_set->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pe_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] data_set Cluster working set
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pe_working_set_t *data_set)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, data_set);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] data_set Cluster working set
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pe_working_set_t *data_set)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, data_set);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, XML_LRM_ATTR_TASK);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] data_set Cluster working set
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pe_working_set_t *data_set)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, data_set)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	data_set);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pe_node_t source_node,
	const pe_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	crm_err("Ignoring resource history entry %s without "
	XML_LRM_ATTR_MIGRATE_SOURCE " and " XML_LRM_ATTR_MIGRATE_TARGET,
	ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	crm_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	ID(entry), *source_name, pe__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	crm_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	ID(entry), *target_name, pe__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pe_resource_t rsc, const pe_node_t node)
	{
	pe_rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pe__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pe_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
	&from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pe_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pe_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pe_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pe__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pe__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pe_resource_t *rsc)
	{
	int score = -INFINITY;
	pe_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pe_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[out] last_failure Set this to action XML
	* \param[in,out] on_fail What should be done about the result
	*/
	static void
	unpack_rsc_op_failure(struct action_history history, xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	pe_action_t *action = NULL;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	crm_warn("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	history->rsc->id, pe__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	action = custom_action(history->rsc, strdup(history->key), history->task,
	NULL, TRUE, FALSE, history->rsc->cluster);
	if (cmp_on_fail(*on_fail, action->on_fail) < 0) {
	pe_rsc_trace(history->rsc, "on-fail %s -> %s for %s (%s)",
	fail2text(*on_fail), fail2text(action->on_fail),
	action->uuid, history->key);
	*on_fail = action->on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (action->on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pe_rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pe_rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pe_rsc_trace(history->rsc,
	"Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(action->on_fail), role2text(action->fail_role));

	if ((action->fail_role != pcmk_role_started)
	&& (history->rsc->next_role < action->fail_role)) {
	pe__set_next_role(history->rsc, action->fail_role, "failure");
	}

	if (action->fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}

	pe_free_action(action);
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pe_proc_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task, pe__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pe__clear_resource_flags(history->rsc, pcmk_rsc_managed);
	pe__set_resource_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the data set's list of failed operations
	* to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	crm_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pe__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pe_rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pe__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pe_rsc_info(history->rsc, "Probe found %s active on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pe_rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pe__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pe_rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pe_resource_t rsc, pe_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	op_digest_cache_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pe__xe_history_key(xml_op),
	node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pe_action_t action, pe_resource_t remote_conn,
	pe_working_set_t *data_set)
	{
	pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);

	if (remote_node) {
	pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, data_set);

	- order_actions(fence, action, pe_order_implies_then);
	+ order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pe_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pe_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pe_rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pe__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	// Resource has a failure-timeout, and history entry has a timestamp

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure, pe_fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster);
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL, pe_fc_default,
	history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pe_rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Unexpired fail count",
	history->id, pe__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	// Schedule clearing of the fail count
	pe_action_t *clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason,
	history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pe_rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Probe result",
	history->id, pe__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Get the failure handling for an action
	*
	* \param[in,out] history Parsed action history entry
	*
	* \return Failure handling appropriate to action
	*/
	static enum action_fail_response
	get_action_on_fail(struct action_history *history)
	{
	enum action_fail_response result = pcmk_on_fail_restart;
	pe_action_t *action = custom_action(history->rsc, strdup(history->key),
	history->task, NULL, TRUE, FALSE,
	history->rsc->cluster);

	result = action->on_fail;
	pe_free_action(action);
	return result;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pe_rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pe__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pe_rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail), history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	crm_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	history->id, history->rsc->id, pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_OPSTATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	crm_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_RC " '%s'",
	history->id, history->rsc->id, pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_RC),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml,
	XML_LRM_ATTR_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pe_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pe__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, XML_LRM_ATTR_TASK),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	XML_RSC_OP_LAST_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pe__set_resource_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pe_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	XML_LRM_ATTR_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pe_resource_t rsc, pe_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pe_resource_t *parent = rsc;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	crm_err("Ignoring resource history entry for %s on %s without ID",
	rsc->id, pe__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (history.task == NULL) {
	crm_err("Ignoring resource history entry %s for %s on %s without "
	XML_LRM_ATTR_TASK, history.id, rsc->id, pe__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
	&(history.interval_ms));
	if (!can_affect_state(&history)) {
	pe_rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pe__node_name(node), history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pe__xe_history_key(xml_op);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));

	pe_rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id, pe__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pe_rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pe__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	failure_strategy = get_action_on_fail(&history);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pe__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	failure_strategy = get_action_on_fail(&history);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
	last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	pe__set_resource_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pe__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	crm_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " %s",
	parent->id, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pe_rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pe__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pe_node_t node, bool overwrite,
	pe_working_set_t *data_set)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, data_set->dc_uuid, pcmk__str_casei)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite, data_set);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, data_set);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pe_working_set_t * data_set)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	pe_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);

	} else {
	determine_online_status(node_state, this_node, data_set);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(data_set->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 9:55 AM (1 d, 1 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1664770
Default Alt Text: (626 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions