No OneTemporary
Actions

Size

451 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
	index 495bd966ce..9eafc3285e 100644
	--- a/include/crm/pengine/pe_types.h
	+++ b/include/crm/pengine/pe_types.h
	@@ -1,442 +1,441 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_PE_TYPES__H
	# define PCMK__CRM_PENGINE_PE_TYPES__H


	# include <stdbool.h> // bool
	# include <sys/types.h> // time_t
	# include <libxml/tree.h> // xmlNode
	# include <glib.h> // gboolean, guint, GList, GHashTable
	# include <crm/common/iso8601.h>
	# include <crm/common/scheduler.h>
	# include <crm/pengine/common.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Data types for cluster status
	* \ingroup pengine
	*/

	typedef struct pe_node_s pe_node_t;
	typedef struct pe_action_s pe_action_t;
	typedef struct pe_resource_s pe_resource_t;
	typedef struct pe_working_set_s pe_working_set_t;

	typedef struct resource_object_functions_s {
	gboolean (unpack) (pe_resource_t, pe_working_set_t*);
	pe_resource_t (find_rsc) (pe_resource_t parent, const char search,
	const pe_node_t *node, int flags);
	/* parameter result must be free'd */
	char (parameter) (pe_resource_t, pe_node_t, gboolean, const char*,
	pe_working_set_t*);
	//! \deprecated will be removed in a future release
	void (print) (pe_resource_t, const char, long, void);
	gboolean (active) (pe_resource_t, gboolean);
	enum rsc_role_e (state) (const pe_resource_t, gboolean);
	pe_node_t (location) (const pe_resource_t, GList*, int);
	void (free) (pe_resource_t);
	void (count) (pe_resource_t);
	gboolean (is_filtered) (const pe_resource_t, GList *, gboolean);

	/*!
	* \brief Find a node (and optionally count all) where resource is active
	*
	* \param[in] rsc Resource to check
	* \param[out] count_all If not NULL, set this to count of active nodes
	* \param[out] count_clean If not NULL, set this to count of clean nodes
	*
	* \return A node where the resource is active, preferring the source node
	* if the resource is involved in a partial migration or a clean,
	* online node if the resource's "requires" is "quorum" or
	* "nothing", or NULL if the resource is inactive.
	*/
	pe_node_t (active_node)(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean);

	/*!
	* \brief Get maximum resource instances per node
	*
	* \param[in] rsc Resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int (max_per_node)(const pe_resource_t rsc);
	} resource_object_functions_t;

	typedef struct resource_alloc_functions_s resource_alloc_functions_t;

	struct pe_working_set_s {
	xmlNode *input;
	crm_time_t *now;

	/* options extracted from the input */
	char *dc_uuid;
	pe_node_t *dc_node;
	const char *stonith_action;
	const char *placement_strategy;

	unsigned long long flags;

	int stonith_timeout;
	enum pe_quorum_policy no_quorum_policy;

	GHashTable *config_hash;
	GHashTable *tickets;

	// Actions for which there can be only one (e.g. fence nodeX)
	GHashTable *singletons;

	GList *nodes;
	GList *resources;
	GList *placement_constraints;
	GList *ordering_constraints;
	GList *colocation_constraints;
	GList *ticket_constraints;

	GList *actions;
	xmlNode *failed;
	xmlNode *op_defaults;
	xmlNode *rsc_defaults;

	/* stats */
	int num_synapse;
	int max_valid_nodes; //! Deprecated (will be removed in a future release)
	int order_id;
	int action_id;

	/* final output */
	xmlNode *graph;

	GHashTable *template_rsc_sets;
	const char *localhost;
	GHashTable *tags;

	int blocked_resources;
	int disabled_resources;

	GList *param_check; // History entries that need to be checked
	GList *stop_needed; // Containers that need stop actions
	time_t recheck_by; // Hint to controller to re-run scheduler by this time
	int ninstances; // Total number of resource instances
	guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
	int priority_fencing_delay; // Priority fencing delay

	void *priv;
	guint node_pending_timeout; // Node pending timeout
	};

	struct pe_node_shared_s {
	const char *id;
	const char *uname;
	enum node_type type;

	/* @TODO convert these flags into a bitfield */
	gboolean online;
	gboolean standby;
	gboolean standby_onfail;
	gboolean pending;
	gboolean unclean;
	gboolean unseen;
	gboolean shutdown;
	gboolean expected_up;
	gboolean is_dc;
	gboolean maintenance;
	gboolean rsc_discovery_enabled;
	gboolean remote_requires_reset;
	gboolean remote_was_fenced;
	gboolean remote_maintenance; /* what the remote-rsc is thinking */
	gboolean unpacked;

	int num_resources;
	pe_resource_t *remote_rsc;
	GList running_rsc; / pe_resource_t* */
	GList allocated_rsc; / pe_resource_t* */

	GHashTable attrs; / char* => char* */
	GHashTable *utilization;
	GHashTable *digest_cache; //!< cache of calculated resource digests
	int priority; // calculated based on the priority of resources running on the node
	pe_working_set_t *data_set; //!< Cluster that this node is part of
	};

	struct pe_node_s {
	int weight;
	gboolean fixed; //!< \deprecated Will be removed in a future release
	int count;
	struct pe_node_shared_s *details;
	int rsc_discover_mode;
	};

	-# define pe_rsc_maintenance 0x04000000ULL
	# define pe_rsc_is_container 0x08000000ULL

	# define pe_rsc_needs_quorum 0x10000000ULL
	# define pe_rsc_needs_fencing 0x20000000ULL
	# define pe_rsc_needs_unfencing 0x40000000ULL

	/* INDENT-OFF */
	enum pe_action_flags {
	pe_action_pseudo = 0x00001,
	pe_action_runnable = 0x00002,
	pe_action_optional = 0x00004,
	pe_action_print_always = 0x00008,

	pe_action_have_node_attrs = 0x00010,
	pe_action_implied_by_stonith = 0x00040,
	pe_action_migrate_runnable = 0x00080,

	pe_action_dumped = 0x00100,
	pe_action_processed = 0x00200,
	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	pe_action_clear = 0x00400, //! \deprecated Unused
	#endif
	pe_action_dangle = 0x00800,

	/* This action requires one or more of its dependencies to be runnable.
	* We use this to clear the runnable flag before checking dependencies.
	*/
	pe_action_requires_any = 0x01000,

	pe_action_reschedule = 0x02000,
	pe_action_tracking = 0x04000,
	pe_action_dedup = 0x08000, //! Internal state tracking when creating graph

	pe_action_dc = 0x10000, //! Action may run on DC instead of target
	};
	/* INDENT-ON */

	struct pe_resource_s {
	char *id;
	char *clone_name;
	xmlNode *xml;
	xmlNode *orig_xml;
	xmlNode *ops_xml;

	pe_working_set_t *cluster;
	pe_resource_t *parent;

	enum pe_obj_types variant;
	void *variant_opaque;
	resource_object_functions_t *fns;
	resource_alloc_functions_t *cmds;

	enum rsc_recovery_type recovery_type;

	enum pe_restart restart_type; //!< \deprecated will be removed in future release

	int priority;
	int stickiness;
	int sort_index;
	int failure_timeout;
	int migration_threshold;
	guint remote_reconnect_ms;
	char *pending_task;

	unsigned long long flags;

	// @TODO merge these into flags
	gboolean is_remote_node;
	gboolean exclusive_discover;

	/* Pay special attention to whether you want to use rsc_cons_lhs and
	* rsc_cons directly, which include only colocations explicitly involving
	* this resource, or call libpacemaker's pcmk__with_this_colocations() and
	* pcmk__this_with_colocations() functions, which may return relevant
	* colocations involving the resource's ancestors as well.
	*/

	//!@{
	//! This field should be treated as internal to Pacemaker
	GList rsc_cons_lhs; // List of pcmk__colocation_t
	GList rsc_cons; // List of pcmk__colocation_t
	GList rsc_location; // List of pe__location_t
	GList actions; // List of pe_action_t
	GList rsc_tickets; // List of rsc_ticket
	//!@}

	pe_node_t *allocated_to;
	pe_node_t *partial_migration_target;
	pe_node_t *partial_migration_source;
	GList running_on; / pe_node_t* */
	GHashTable known_on; / pe_node_t* */
	GHashTable allowed_nodes; / pe_node_t* */

	enum rsc_role_e role;
	enum rsc_role_e next_role;

	GHashTable *meta;
	GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
	GHashTable *utilization;

	GList children; / pe_resource_t* */
	GList dangling_migrations; / pe_node_t* */

	pe_resource_t *container;
	GList *fillers;

	// @COMPAT These should be made const at next API compatibility break
	pe_node_t *pending_node; // Node on which pending_task is happening
	pe_node_t *lock_node; // Resource is shutdown-locked to this node

	time_t lock_time; // When shutdown lock started

	/* Resource parameters may have node-attribute-based rules, which means the
	* values can vary by node. This table is a cache of parameter name/value
	* tables for each node (as needed). Use pe_rsc_params() to get the table
	* for a given node.
	*/
	GHashTable *parameter_cache; // Key = node name, value = parameters table
	};

	struct pe_action_s {
	int id;
	int priority;

	pe_resource_t *rsc;
	pe_node_t *node;
	xmlNode *op_entry;

	char *task;
	char *uuid;
	char *cancel_task;
	char *reason;

	enum pe_action_flags flags;
	enum rsc_start_requirement needs;
	enum action_fail_response on_fail;
	enum rsc_role_e fail_role;

	GHashTable *meta;
	GHashTable *extra;

	/*
	* These two varables are associated with the constraint logic
	* that involves first having one or more actions runnable before
	* then allowing this action to execute.
	*
	* These varables are used with features such as 'clone-min' which
	* requires at minimum X number of cloned instances to be running
	* before an order dependency can run. Another option that uses
	* this is 'require-all=false' in ordering constrants. This option
	* says "only require one instance of a resource to start before
	* allowing dependencies to start" -- basically, require-all=false is
	* the same as clone-min=1.
	*/

	/* current number of known runnable actions in the before list. */
	int runnable_before;
	/* the number of "before" runnable actions required for this action
	* to be considered runnable */
	int required_runnable_before;

	GList actions_before; / pe_action_wrapper_t* */
	GList actions_after; / pe_action_wrapper_t* */

	/* Some of the above fields could be moved to the details,
	* except for API backward compatibility.
	*/
	void *action_details; // varies by type of action
	};

	typedef struct pe_ticket_s {
	char *id;
	gboolean granted;
	time_t last_granted;
	gboolean standby;
	GHashTable *state;
	} pe_ticket_t;

	typedef struct pe_tag_s {
	char *id;
	GList *refs;
	} pe_tag_t;

	//! Internal tracking for transition graph creation
	enum pe_link_state {
	pe_link_not_dumped, //! Internal tracking for transition graph creation
	pe_link_dumped, //! Internal tracking for transition graph creation
	pe_link_dup, //! \deprecated No longer used by Pacemaker
	};

	enum pe_discover_e {
	pe_discover_always = 0,
	pe_discover_never,
	pe_discover_exclusive,
	};

	/* INDENT-OFF */
	enum pe_ordering {
	pe_order_none = 0x0, /* deleted */
	pe_order_optional = 0x1, /* pure ordering, nothing implied */
	pe_order_apply_first_non_migratable = 0x2, /* Only apply this constraint's ordering if first is not migratable. */

	pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */
	pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */
	pe_order_promoted_implies_first = 0x40, /* If 'then' is required and then's rsc is promoted, ensure 'first' becomes required too */

	/* first requires then to be both runnable and migrate runnable. */
	pe_order_implies_first_migratable = 0x80,

	pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */

	pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */
	pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX',
	* ensure instances of 'then' on 'nodeX' are too.
	* Only really useful if 'then' is a clone and 'first' is not
	*/
	pe_order_probe = 0x800, /* If 'first->rsc' is
	* - running but about to stop, ignore the constraint
	* - otherwise, behave as runnable_left
	*/

	pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */
	pe_order_stonith_stop = 0x2000, //<! \deprecated Will be removed in future release
	pe_order_serialize_only = 0x4000, /* serialize */
	pe_order_same_node = 0x8000, /* applies only if 'first' and 'then' are on same node */

	pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
	pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not mandatory */

	pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */
	pe_order_load = 0x200000, /* Only relevant if... */
	pe_order_one_or_more = 0x400000, /* 'then' is runnable only if one or more of its dependencies are too */
	pe_order_anti_colocation = 0x800000,

	pe_order_preserve = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
	pe_order_then_cancels_first = 0x2000000, // if 'then' becomes required, 'first' becomes optional
	pe_order_trace = 0x4000000, /* test marker */

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	// \deprecated Use pe_order_promoted_implies_first instead
	pe_order_implies_first_master = pe_order_promoted_implies_first,
	#endif
	};
	/* INDENT-ON */

	typedef struct pe_action_wrapper_s {
	enum pe_ordering type;
	enum pe_link_state state;
	pe_action_t *action;
	} pe_action_wrapper_t;

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/pengine/pe_types_compat.h>
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_PENGINE_PE_TYPES__H
	diff --git a/include/crm/pengine/pe_types_compat.h b/include/crm/pengine/pe_types_compat.h
	index 3bc438b119..bb0954c9b6 100644
	--- a/include/crm/pengine/pe_types_compat.h
	+++ b/include/crm/pengine/pe_types_compat.h
	@@ -1,224 +1,227 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	# define PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H

	#include <crm/pengine/pe_types.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker scheduler API
	* \ingroup pengine
	* \deprecated Do not include this header directly. The scheduler APIs in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	//! \deprecated Use pcmk_rsc_removed instead
	#define pe_rsc_orphan pcmk_rsc_removed

	//! \deprecated Use pcmk_rsc_managed instead
	#define pe_rsc_managed pcmk_rsc_managed

	//! \deprecated Use pcmk_rsc_blocked instead
	#define pe_rsc_block pcmk_rsc_blocked

	//! \deprecated Use pcmk_rsc_removed_filler instead
	#define pe_rsc_orphan_container_filler pcmk_rsc_removed_filler

	//! \deprecated Use pcmk_rsc_notify instead
	#define pe_rsc_notify pcmk_rsc_notify

	//! \deprecated Use pcmk_rsc_unique instead
	#define pe_rsc_unique pcmk_rsc_unique

	//! \deprecated Use pcmk_rsc_fence_device instead
	#define pe_rsc_fence_device pcmk_rsc_fence_device

	//! \deprecated Use pcmk_rsc_promotable instead
	#define pe_rsc_promotable pcmk_rsc_promotable

	//! \deprecated Use pcmk_rsc_unassigned instead
	#define pe_rsc_provisional pcmk_rsc_unassigned

	//! \deprecated Use pcmk_rsc_assigning instead
	#define pe_rsc_allocating pcmk_rsc_assigning

	//! \deprecated Use pcmk_rsc_updating_nodes instead
	#define pe_rsc_merging pcmk_rsc_updating_nodes

	//! \deprecated Use pcmk_rsc_restarting instead
	#define pe_rsc_restarting pcmk_rsc_restarting

	//! \deprecated Use pcmk_rsc_stop_if_failed instead
	#define pe_rsc_stop pcmk_rsc_stop_if_failed

	//! \deprecated Use pcmk_rsc_reload instead
	#define pe_rsc_reload pcmk_rsc_reload

	//! \deprecated Use pcmk_rsc_remote_nesting_allowed instead
	#define pe_rsc_allow_remote_remotes pcmk_rsc_remote_nesting_allowed

	//! \deprecated Use pcmk_rsc_critical instead
	#define pe_rsc_critical pcmk_rsc_critical

	//! \deprecated Use pcmk_rsc_failed instead
	#define pe_rsc_failed pcmk_rsc_failed

	//! \deprecated Use pcmk_rsc_detect_loop instead
	#define pe_rsc_detect_loop pcmk_rsc_detect_loop

	//! \deprecated Do not use
	#define pe_rsc_runnable pcmk_rsc_runnable

	//! \deprecated Use pcmk_rsc_start_pending instead
	#define pe_rsc_start_pending pcmk_rsc_start_pending

	//!< \deprecated Do not use
	#define pe_rsc_starting pcmk_rsc_starting

	//!< \deprecated Do not use
	#define pe_rsc_stopping pcmk_rsc_stopping

	//! \deprecated Use pcmk_rsc_stop_unexpected instead
	#define pe_rsc_stop_unexpected pcmk_rsc_stop_unexpected

	//! \deprecated Use pcmk_rsc_migratable instead
	#define pe_rsc_allow_migrate pcmk_rsc_migratable

	//! \deprecated Use pcmk_rsc_ignore_failure instead
	#define pe_rsc_failure_ignored pcmk_rsc_ignore_failure

	//! \deprecated Use pcmk_rsc_replica_container instead
	#define pe_rsc_replica_container pcmk_rsc_replica_container

	+//! \deprecated Use pcmk_rsc_maintenance instead
	+#define pe_rsc_maintenance pcmk_rsc_maintenance
	+
	//! \deprecated Use pcmk_sched_quorate instead
	#define pe_flag_have_quorum pcmk_sched_quorate

	//! \deprecated Use pcmk_sched_symmetric_cluster instead
	#define pe_flag_symmetric_cluster pcmk_sched_symmetric_cluster

	//! \deprecated Use pcmk_sched_in_maintenance instead
	#define pe_flag_maintenance_mode pcmk_sched_in_maintenance

	//! \deprecated Use pcmk_sched_fencing_enabled instead
	#define pe_flag_stonith_enabled pcmk_sched_fencing_enabled

	//! \deprecated Use pcmk_sched_have_fencing instead
	#define pe_flag_have_stonith_resource pcmk_sched_have_fencing

	//! \deprecated Use pcmk_sched_enable_unfencing instead
	#define pe_flag_enable_unfencing pcmk_sched_enable_unfencing

	//! \deprecated Use pcmk_sched_concurrent_fencing instead
	#define pe_flag_concurrent_fencing pcmk_sched_concurrent_fencing

	//! \deprecated Use pcmk_sched_stop_removed_resources instead
	#define pe_flag_stop_rsc_orphans pcmk_sched_stop_removed_resources

	//! \deprecated Use pcmk_sched_cancel_removed_actions instead
	#define pe_flag_stop_action_orphans pcmk_sched_cancel_removed_actions

	//! \deprecated Use pcmk_sched_stop_all instead
	#define pe_flag_stop_everything pcmk_sched_stop_all

	//! \deprecated Use pcmk_sched_start_failure_fatal instead
	#define pe_flag_start_failure_fatal pcmk_sched_start_failure_fatal

	//! \deprecated Do not use
	#define pe_flag_remove_after_stop pcmk_sched_remove_after_stop

	//! \deprecated Use pcmk_sched_startup_fencing instead
	#define pe_flag_startup_fencing pcmk_sched_startup_fencing

	//! \deprecated Use pcmk_sched_shutdown_lock instead
	#define pe_flag_shutdown_lock pcmk_sched_shutdown_lock

	//! \deprecated Use pcmk_sched_probe_resources instead
	#define pe_flag_startup_probes pcmk_sched_probe_resources

	//! \deprecated Use pcmk_sched_have_status instead
	#define pe_flag_have_status pcmk_sched_have_status

	//! \deprecated Use pcmk_sched_have_remote_nodes instead
	#define pe_flag_have_remote_nodes pcmk_sched_have_remote_nodes

	//! \deprecated Use pcmk_sched_location_only instead
	#define pe_flag_quick_location pcmk_sched_location_only

	//! \deprecated Use pcmk_sched_sanitized instead
	#define pe_flag_sanitized pcmk_sched_sanitized

	//! \deprecated Do not use
	#define pe_flag_stdout (1ULL << 22)

	//! \deprecated Use pcmk_sched_no_counts instead
	#define pe_flag_no_counts pcmk_sched_no_counts

	//! \deprecated Use pcmk_sched_no_compat instead
	#define pe_flag_no_compat pcmk_sched_no_compat

	//! \deprecated Use pcmk_sched_output_scores instead
	#define pe_flag_show_scores pcmk_sched_output_scores

	//! \deprecated Use pcmk_sched_show_utilization instead
	#define pe_flag_show_utilization pcmk_sched_show_utilization

	//! \deprecated Use pcmk_sched_validate_only instead
	#define pe_flag_check_config pcmk_sched_validate_only

	//!@{
	//! \deprecated Do not use (unused by Pacemaker)
	enum pe_graph_flags {
	pe_graph_none = 0x00000,
	pe_graph_updated_first = 0x00001,
	pe_graph_updated_then = 0x00002,
	pe_graph_disable = 0x00004,
	};
	//!@}

	//!@{
	//! \deprecated Do not use
	enum pe_check_parameters {
	pe_check_last_failure,
	pe_check_active,
	};
	//!@}

	//!< \deprecated Use pe_action_t instead
	typedef struct pe_action_s action_t;

	//!< \deprecated Use pe_action_wrapper_t instead
	typedef struct pe_action_wrapper_s action_wrapper_t;

	//!< \deprecated Use pe_node_t instead
	typedef struct pe_node_s node_t;

	//!< \deprecated Use enum pe_quorum_policy instead
	typedef enum pe_quorum_policy no_quorum_policy_t;

	//!< \deprecated use pe_resource_t instead
	typedef struct pe_resource_s resource_t;

	//!< \deprecated Use pe_tag_t instead
	typedef struct pe_tag_s tag_t;

	//!< \deprecated Use pe_ticket_t instead
	typedef struct pe_ticket_s ticket_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
	index 6dcd0195a3..509671f942 100644
	--- a/lib/pacemaker/pcmk_graph_producer.c
	+++ b/lib/pacemaker/pcmk_graph_producer.c
	@@ -1,1095 +1,1095 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pe_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, XML_ATTR_ID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pe_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Count (optionally add to XML) nodes needing maintenance state update
	*
	* \param[in,out] xml Parent XML tag to add to, if any
	* \param[in] data_set Working set for cluster
	*
	* \return Count of nodes added
	* \note Only Pacemaker Remote nodes are considered currently
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pe_working_set_t data_set)
	{
	xmlNode *maintenance = NULL;
	int count = 0;

	if (xml != NULL) {
	maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
	}
	for (const GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	const pe_node_t *node = iter->data;

	if (pe__is_guest_or_remote_node(node) &&
	(node->details->maintenance != node->details->remote_maintenance)) {

	if (maintenance != NULL) {
	crm_xml_add(add_node_to_xml_by_id(node->details->id,
	maintenance),
	XML_NODE_IS_MAINTENANCE,
	(node->details->maintenance? "1" : "0"));
	}
	count++;
	}
	}
	crm_trace("%s %d nodes in need of maintenance mode update in state",
	((maintenance == NULL)? "Counted" : "Added"), count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] data_set Working set for cluster
	*/
	static void
	add_maintenance_update(pe_working_set_t *data_set)
	{
	pe_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, data_set) != 0) {
	action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, data_set);
	pe__set_action_flags(action, pe_action_print_always);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	*/
	static void
	add_downed_nodes(xmlNode xml, const pe_action_t action)
	{
	CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
	return);

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__is_fencing_action(fence)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(action->node->details->data_set, action->node,
	add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pe_action_t *input;
	bool migrating = false;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pe_action_wrapper_t *) iter->data)->action;
	if ((input->rsc != NULL)
	&& pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
	&& pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	migrating = true;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph operation key for a clone action
	*
	* \param[in] action Clone action
	* \param[in] interval_ms Action interval in milliseconds
	*
	* \return Newly allocated string with transition graph operation key
	*/
	static char *
	clone_op_key(const pe_action_t *action, guint interval_ms)
	{
	if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_operation");

	CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
	return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);

	} else if (action->cancel_task != NULL) {
	return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
	interval_ms);
	} else {
	return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
	}
	}

	/*!
	* \internal
	* \brief Add node details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] xml Transition graph action XML for \p action
	*/
	static void
	add_node_details(const pe_action_t action, xmlNode xml)
	{
	pe_node_t *router_node = pcmk__connection_host_for_action(action);

	crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
	crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
	if (router_node != NULL) {
	crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Add resource details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_resource_details(const pe_action_t action, xmlNode action_xml)
	{
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER,
	XML_ATTR_TYPE
	};

	/* If a resource is locked to a node via shutdown-lock, mark its actions
	* so the controller can preserve the lock when the action completes.
	*/
	if (pcmk__action_locks_rsc_to_node(action)) {
	crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	(const char *) action->rsc->xml->name);
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed)
	&& (action->rsc->clone_name != NULL)) {
	/* Use the numbered instance name here, because if there is more
	* than one instance on a node, we need to make sure the command
	* goes to the right one.
	*
	* This is important even for anonymous clones, because the clone's
	* unique meta-attribute might have just been toggled from on to
	* off.
	*/
	crm_debug("Using orphan clone name %s instead of %s",
	action->rsc->id, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka %s)",
	xml_id, action->rsc->id, action->rsc->clone_name);

	/* ID is what we'd like client to use
	* ID_LONG is what they might know it as instead
	*
	* ID_LONG is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
	if ((action->rsc->clone_name != NULL)
	&& !pcmk__str_eq(xml_id, action->rsc->clone_name,
	pcmk__str_none)) {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
	}

	for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/*!
	* \internal
	* \brief Add action attributes to transition graph action XML
	*
	* \param[in,out] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_action_attributes(pe_action_t action, xmlNode action_xml)
	{
	xmlNode *args_xml = NULL;

	/* We create free-standing XML to start, so we can sort the attributes
	* before adding it to action_xml, which keeps the scheduler regression
	* test graphs comparable.
	*/
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);

	crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	g_hash_table_foreach(action->extra, hash2field, args_xml);

	if ((action->rsc != NULL) && (action->node != NULL)) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node,
	action->rsc->cluster);

	pcmk__substitute_remote_addr(action->rsc, params);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	} else if ((action->rsc != NULL)
	&& (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL,
	action->rsc->cluster);

	g_hash_table_foreach(params, hash2smartfield, args_xml);
	}

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	pe_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->add_graph_meta(parent, args_xml);
	parent = parent->parent;
	}

	pcmk__add_bundle_meta_to_xml(args_xml, action);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& (action->node != NULL)) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield,
	args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);
	}

	/*!
	* \internal
	* \brief Create the transition graph XML for a scheduled action
	*
	* \param[in,out] parent Parent XML element to add action to
	* \param[in,out] action Scheduled action
	* \param[in] skip_details If false, add action details as sub-elements
	* \param[in] data_set Cluster working set
	*/
	static void
	create_graph_action(xmlNode parent, pe_action_t action, bool skip_details,
	const pe_working_set_t *data_set)
	{
	bool needs_node_info = true;
	bool needs_maintenance_info = false;
	xmlNode *action_xml = NULL;

	if ((action == NULL) \|\| (data_set == NULL)) {
	return;
	}

	// Create the top-level element based on task

	if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
	/* All fences need node info; guest node fences are pseudo-events */
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	}

	} else if (pcmk__str_any_of(action->task,
	PCMK_ACTION_DO_SHUTDOWN,
	PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
	pcmk__str_none)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
	pcmk__str_none)) {
	needs_maintenance_info = true;
	}
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = false;

	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
	}

	crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);

	if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}
	clone_key = clone_op_key(action, interval_ms);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY,
	action->uuid);
	free(clone_key);
	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if (needs_node_info && (action->node != NULL)) {
	add_node_details(action, action_xml);
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
	strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
	strdup(action->node->details->id));
	}

	if (skip_details) {
	return;
	}

	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->flags, pe_action_pseudo)) {

	// This is a real resource action, so add resource details
	add_resource_details(action, action_xml);
	}

	/* List any attributes in effect */
	add_action_attributes(action, action_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, data_set);
	}
	}

	/*!
	* \internal
	* \brief Check whether an action should be added to the transition graph
	*
	* \param[in] action Action to check
	*
	* \return true if action should be added to graph, otherwise false
	*/
	static bool
	should_add_action_to_graph(const pe_action_t *action)
	{
	if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;
	}

	if (pcmk_is_set(action->flags, pe_action_optional)
	&& !pcmk_is_set(action->flags, pe_action_print_always)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;
	}

	/* Actions for unmanaged resources should be excluded from the graph,
	* with the exception of monitors and cancellation of recurring monitors.
	*/
	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {

	const char *interval_ms_s;

	/* A cancellation of a recurring monitor will get here because the task
	* is cancel rather than monitor, but the interval can still be used to
	* recognize it. The interval has been normalized to milliseconds by
	* this point, so a string comparison is sufficient.
	*/
	interval_ms_s = g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	/* Always add pseudo-actions, fence actions, and shutdown actions (already
	* determined to be required and runnable by this point)
	*/
	if (pcmk_is_set(action->flags, pe_action_pseudo)
	\|\| pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
	PCMK_ACTION_DO_SHUTDOWN, NULL)) {
	return true;
	}

	if (action->node == NULL) {
	pe_err("Skipping action %s (%d) "
	"because it was not assigned to a node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unassigned", action, false);
	return false;
	}

	if (pcmk_is_set(action->flags, pe_action_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (!action->node->details->online) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for offline node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Offline node", action, false);
	return false;

	} else if (action->node->details->unclean) {
	pe_err("Skipping action %s (%d) "
	"because it was scheduled for unclean node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unclean node", action, false);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering's flags can change an action
	*
	* \param[in] ordering Ordering to check
	*
	* \return true if ordering has flags that can change an action, false otherwise
	*/
	static bool
	ordering_can_change_actions(const pe_action_wrapper_t *ordering)
	{
	return pcmk_any_flags_set(ordering->type, ~(pe_order_implies_first_printed
	\|pe_order_implies_then_printed
	\|pe_order_optional));
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	should_add_input_to_graph(const pe_action_t action, pe_action_wrapper_t input)
	{
	if (input->state == pe_link_dumped) {
	return true;
	}

	if (input->type == pe_order_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& !ordering_can_change_actions(input)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
	&& pcmk_is_set(input->type, pe_order_one_or_more)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"one-or-more and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
	&& !pcmk_is_set(input->action->flags, pe_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"implies input migratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
	&& pcmk_is_set(input->action->flags,
	pe_action_migrate_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"only if input unmigratable but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((input->type == pe_order_optional)
	&& pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (input->type == pe_order_load) {
	pe_node_t *input_node = input->action->node;

	// load orderings are relevant only if actions are for same node

	if ((action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {

	pe_node_t *assigned = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where it
	* has been assigned to, not where it will be executed.
	*/
	if (!pe__same_node(input_node, assigned)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(assigned? assigned->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;
	}

	} else if (!pe__same_node(input_node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"load ordering input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->type == pe_order_anti_colocation) {
	if (input->action->node && action->node
	&& !pe__same_node(input->action->node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation node mismatch %s vs %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	pe__node_name(action->node),
	pe__node_name(input->action->node));
	input->type = pe_order_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"anti-colocation input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = pe_order_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pe_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pe_action_print_always\|pe_action_dumped)
	&& !should_add_action_to_graph(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
	action->uuid, action->id, action_type_str(input->action->flags),
	input->action->uuid, input->action->id,
	action_node_str(input->action),
	action_runnable_str(input->action->flags),
	action_optional_str(input->action->flags), input->type);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering creates an ordering loop
	*
	* \param[in] init_action "First" action in ordering
	* \param[in] action Callers should always set this the same as
	* \p init_action (this function may use a different
	* value for recursive calls)
	* \param[in,out] input Action wrapper for "then" action in ordering
	*
	* \return true if the ordering creates a loop, otherwise false
	*/
	bool
	pcmk__graph_has_loop(const pe_action_t init_action, const pe_action_t action,
	pe_action_wrapper_t *input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!should_add_input_to_graph(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pe__set_action_flags(input->action, pe_action_tracking);

	crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (pcmk__graph_has_loop(init_action, input->action,
	(pe_action_wrapper_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	break;
	}
	}

	pe__clear_action_flags(input->action, pe_action_tracking);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	/*!
	* \internal
	* \brief Create a synapse XML element for a transition graph
	*
	* \param[in] action Action that synapse is for
	* \param[in,out] data_set Cluster working set containing graph
	*
	* \return Newly added XML element for new graph synapse
	*/
	static xmlNode *
	create_graph_synapse(const pe_action_t action, pe_working_set_t data_set)
	{
	int synapse_priority = 0;
	xmlNode *syn = create_xml_node(data_set->graph, "synapse");

	crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
	data_set->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}
	return syn;
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in,out] data Action to possibly add
	* \param[in,out] user_data Cluster working set
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pe_action_wrapper_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after
	* pcmk__apply_orderings() is complete, and nothing after this should rely
	* on those type flags. (For example, some code looks for type equal to
	* some flag rather than whether the flag is set, and some code looks for
	* particular combinations of flags -- such code must be done before
	* pcmk__create_graph().)
	*/
	static void
	add_action_to_graph(gpointer data, gpointer user_data)
	{
	pe_action_t action = (pe_action_t ) data;
	pe_working_set_t data_set = (pe_working_set_t ) user_data;

	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;

	/* If we haven't already, de-duplicate inputs (even if we won't be adding
	* the action to the graph, so that crm_simulate's dot graphs don't have
	* duplicates).
	*/
	if (!pcmk_is_set(action->flags, pe_action_dedup)) {
	pcmk__deduplicate_action_inputs(action);
	pe__set_action_flags(action, pe_action_dedup);
	}

	if (pcmk_is_set(action->flags, pe_action_dumped) // Already added, or
	\|\| !should_add_action_to_graph(action)) { // shouldn't be added
	return;
	}
	pe__set_action_flags(action, pe_action_dumped);

	crm_trace("Adding action %d (%s%s%s) to graph",
	action->id, action->uuid,
	((action->node == NULL)? "" : " on "),
	((action->node == NULL)? "" : action->node->details->uname));

	syn = create_graph_synapse(action, data_set);
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	create_graph_action(set, action, false, data_set);

	for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) lpc->data;

	if (should_add_input_to_graph(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	create_graph_action(input_xml, input->action, true, data_set);
	}
	}
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__add_rsc_actions_to_graph(pe_resource_t *rsc)
	{
	GList *iter = NULL;

	CRM_ASSERT(rsc != NULL);
	pe_rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);

	// First add the resource's own actions
	g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);

	// Then recursively add its children's actions (appropriate to variant)
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) iter->data;

	child_rsc->cmds->add_actions_to_graph(child_rsc);
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph with all cluster actions needed
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__create_graph(pe_working_set_t *data_set)
	{
	GList *iter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d", transition_id);

	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(data_set->config_hash, "cluster-delay");
	crm_xml_add(data_set->graph, "cluster-delay", value);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	crm_xml_add(data_set->graph, "stonith-timeout", value);

	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(data_set->flags, pcmk_sched_start_failure_fatal)) {
	crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(data_set->graph, "failed-start-offset", "1");
	}

	value = pe_pref(data_set->config_hash, "batch-limit");
	crm_xml_add(data_set->graph, "batch-limit", value);

	crm_xml_add_int(data_set->graph, "transition_id", transition_id);

	value = pe_pref(data_set->config_hash, "migration-limit");
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(data_set->graph, "migration-limit", value);
	}

	if (data_set->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) data_set->recheck_by);
	crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	// Add resource actions to graph
	for (iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	pe_rsc_trace(rsc, "Processing actions for %s", rsc->id);
	rsc->cmds->add_actions_to_graph(rsc);
	}

	// Add pseudo-action for list of nodes with maintenance state update
	add_maintenance_update(data_set);

	// Add non-resource (node) actions
	for (iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;

	if ((action->rsc != NULL)
	&& (action->node != NULL)
	&& action->node->details->shutdown
	- && !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
	+ && !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pe_action_optional\|pe_action_runnable)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* Eventually we should just ignore the 'fence' case, but for now
	* it's the best way to detect (in CTS) when CIB resource updates
	* are being lost.
	*/
	if (pcmk_is_set(data_set->flags, pcmk_sched_quorate)
	\|\| (data_set->no_quorum_policy == pcmk_no_quorum_ignore)) {
	const bool managed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_managed);
	const bool failed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_failed);

	crm_crit("Cannot %s %s because of %s:%s%s (%s)",
	action->node->details->unclean? "fence" : "shut down",
	pe__node_name(action->node), action->rsc->id,
	(managed? " blocked" : " unmanaged"),
	(failed? " failed" : ""), action->uuid);
	}
	}

	add_action_to_graph((gpointer) action, (gpointer) data_set);
	}

	crm_log_xml_trace(data_set->graph, "graph");
	}
	diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
	index 2ae1199700..34db9a7482 100644
	--- a/lib/pacemaker/pcmk_sched_actions.c
	+++ b/lib/pacemaker/pcmk_sched_actions.c
	@@ -1,1930 +1,1930 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/param.h>
	#include <glib.h>

	#include <crm/lrmd_internal.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get the action flags relevant to ordering constraints
	*
	* \param[in,out] action Action to check
	* \param[in] node Node that other action in the ordering is on
	* (used only for clone resource actions)
	*
	* \return Action flags that should be used for orderings
	*/
	static uint32_t
	action_flags_for_ordering(pe_action_t action, const pe_node_t node)
	{
	bool runnable = false;
	uint32_t flags;

	// For non-resource actions, return the action flags
	if (action->rsc == NULL) {
	return action->flags;
	}

	/* For non-clone resources, or a clone action not assigned to a node,
	* return the flags as determined by the resource method without a node
	* specified.
	*/
	flags = action->rsc->cmds->action_flags(action, NULL);
	if ((node == NULL) \|\| !pe_rsc_is_clone(action->rsc)) {
	return flags;
	}

	/* Otherwise (i.e., for clone resource actions on a specific node), first
	* remember whether the non-node-specific action is runnable.
	*/
	runnable = pcmk_is_set(flags, pe_action_runnable);

	// Then recheck the resource method with the node
	flags = action->rsc->cmds->action_flags(action, node);

	/* For clones in ordering constraints, the node-specific "runnable" doesn't
	* matter, just the non-node-specific setting (i.e., is the action runnable
	* anywhere).
	*
	* This applies only to runnable, and only for ordering constraints. This
	* function shouldn't be used for other types of constraints without
	* changes. Not very satisfying, but it's logical and appears to work well.
	*/
	if (runnable && !pcmk_is_set(flags, pe_action_runnable)) {
	pe__set_raw_action_flags(flags, action->rsc->id,
	pe_action_runnable);
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Get action UUID that should be used with a resource ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the UUID and resource of the first action in an
	* ordering, this returns the UUID of the action that should actually be used
	* for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
	*
	* \param[in] first_uuid UUID of first action in ordering
	* \param[in] first_rsc Resource of first action in ordering
	*
	* \return Newly allocated copy of UUID to use with ordering
	* \note It is the caller's responsibility to free the return value.
	*/
	static char *
	action_uuid_for_ordering(const char first_uuid, const pe_resource_t first_rsc)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *first_task_str = NULL;
	enum action_tasks first_task = pcmk_action_unspecified;
	enum action_tasks remapped_task = pcmk_action_unspecified;

	// Only non-notify actions for collective resources need remapping
	if ((strstr(first_uuid, PCMK_ACTION_NOTIFY) != NULL)
	\|\| (first_rsc->variant < pcmk_rsc_variant_group)) {
	goto done;
	}

	// Only non-recurring actions need remapping
	CRM_ASSERT(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
	if (interval_ms > 0) {
	goto done;
	}

	first_task = text2task(first_task_str);
	switch (first_task) {
	case pcmk_action_stop:
	case pcmk_action_start:
	case pcmk_action_notify:
	case pcmk_action_promote:
	case pcmk_action_demote:
	remapped_task = first_task + 1;
	break;
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_notified:
	case pcmk_action_promoted:
	case pcmk_action_demoted:
	remapped_task = first_task;
	break;
	case pcmk_action_monitor:
	case pcmk_action_shutdown:
	case pcmk_action_fence:
	break;
	default:
	crm_err("Unknown action '%s' in ordering", first_task_str);
	break;
	}

	if (remapped_task != pcmk_action_unspecified) {
	/* If a clone or bundle has notifications enabled, the ordering will be
	* relative to when notifications have been sent for the remapped task.
	*/
	if (pcmk_is_set(first_rsc->flags, pcmk_rsc_notify)
	&& (pe_rsc_is_clone(first_rsc) \|\| pe_rsc_is_bundled(first_rsc))) {
	uuid = pcmk__notify_key(rid, "confirmed-post",
	task2text(remapped_task));
	} else {
	uuid = pcmk__op_key(rid, task2text(remapped_task), 0);
	}
	pe_rsc_trace(first_rsc,
	"Remapped action UUID %s to %s for ordering purposes",
	first_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(first_uuid);
	CRM_ASSERT(uuid != NULL);
	}
	free(first_task_str);
	free(rid);
	return uuid;
	}

	/*!
	* \internal
	* \brief Get actual action that should be used with an ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the first action in an ordering, this returns the
	* the action that should actually be used for ordering (for example, the
	* started action instead of the start action).
	*
	* \param[in] action First action in an ordering
	*
	* \return Actual action that should be used for the ordering
	*/
	static pe_action_t *
	action_for_ordering(pe_action_t *action)
	{
	pe_action_t *result = action;
	pe_resource_t *rsc = action->rsc;

	if ((rsc != NULL) && (rsc->variant >= pcmk_rsc_variant_group)
	&& (action->uuid != NULL)) {
	char *uuid = action_uuid_for_ordering(action->uuid, rsc);

	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_warn("Not remapping %s to %s because %s does not have "
	"remapped action", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Wrapper for update_ordered_actions() method for readability
	*
	* \param[in,out] rsc Resource to call method for
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pe_action_optional to affect only
	* mandatory actions, and pe_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pe_ordering flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static inline uint32_t
	update(pe_resource_t rsc, pe_action_t first, pe_action_t *then,
	const pe_node_t *node, uint32_t flags, uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	return rsc->cmds->update_ordered_actions(first, then, node, flags, filter,
	type, data_set);
	}

	/*!
	* \internal
	* \brief Update flags for ordering's actions appropriately for ordering's flags
	*
	* \param[in,out] first First action in an ordering
	* \param[in,out] then Then action in an ordering
	* \param[in] first_flags Action flags for \p first for ordering purposes
	* \param[in] then_flags Action flags for \p then for ordering purposes
	* \param[in,out] order Action wrapper for \p first in ordering
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags
	*/
	static uint32_t
	update_action_for_ordering_flags(pe_action_t first, pe_action_t then,
	uint32_t first_flags, uint32_t then_flags,
	pe_action_wrapper_t *order,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;

	/* The node will only be used for clones. If interleaved, node will be NULL,
	* otherwise the ordering scope will be limited to the node. Normally, the
	* whole 'then' clone should restart if 'first' is restarted, so then->node
	* is needed.
	*/
	pe_node_t *node = then->node;

	if (pcmk_is_set(order->type, pe_order_implies_then_on_node)) {
	/* For unfencing, only instances of 'then' on the same node as 'first'
	* (the unfencing operation) should restart, so reset node to
	* first->node, at which point this case is handled like a normal
	* pe_order_implies_then.
	*/
	pe__clear_order_flags(order->type, pe_order_implies_then_on_node);
	pe__set_order_flags(order->type, pe_order_implies_then);
	node = first->node;
	pe_rsc_trace(then->rsc,
	"%s then %s: mapped pe_order_implies_then_on_node to "
	"pe_order_implies_then on %s",
	first->uuid, then->uuid, pe__node_name(node));
	}

	if (pcmk_is_set(order->type, pe_order_implies_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pe_action_optional,
	pe_action_optional, pe_order_implies_then,
	data_set);
	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)) {
	pe__clear_action_flags(then, pe_action_optional);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_restart) && (then->rsc != NULL)) {
	enum pe_action_flags restart = pe_action_optional\|pe_action_runnable;

	changed \|= update(then->rsc, first, then, node, first_flags, restart,
	pe_order_restart, data_set);
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_restart",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_implies_first)) {
	if (first->rsc != NULL) {
	changed \|= update(first->rsc, first, then, node, first_flags,
	pe_action_optional, pe_order_implies_first,
	data_set);
	} else if (!pcmk_is_set(first_flags, pe_action_optional)
	&& pcmk_is_set(first->flags, pe_action_runnable)) {
	pe__clear_action_flags(first, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_promoted_implies_first)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pe_action_optional,
	pe_action_optional,
	pe_order_promoted_implies_first, data_set);
	}
	pe_rsc_trace(then->rsc,
	"%s then %s: %s after pe_order_promoted_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_one_or_more)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_runnable, pe_order_one_or_more,
	data_set);

	} else if (pcmk_is_set(first_flags, pe_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__set_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_one_or_more",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_probe) && (then->rsc != NULL)) {
	if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& (first->rsc->running_on != NULL)) {

	pe_rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	order->type = pe_order_none;
	} else {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_runnable, pe_order_runnable_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_runnable_left)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_runnable, pe_order_runnable_left,
	data_set);

	} else if (!pcmk_is_set(first_flags, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)) {

	pe__clear_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_runnable_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_migratable)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_optional,
	pe_order_implies_first_migratable, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after "
	"pe_order_implies_first_migratable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_pseudo_left)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_optional, pe_order_pseudo_left,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_pseudo_left",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_optional)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_runnable, pe_order_optional, data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_optional",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pe_order_asymmetrical)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pe_action_runnable, pe_order_asymmetrical,
	data_set);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after pe_order_asymmetrical",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(first->flags, pe_action_runnable)
	&& pcmk_is_set(order->type, pe_order_implies_then_printed)
	&& !pcmk_is_set(first_flags, pe_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pe__set_action_flags(then, pe_action_print_always);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(order->type, pe_order_implies_first_printed)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {

	pe_rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pe__set_action_flags(first, pe_action_print_always);
	// Don't bother marking 'first' as changed just for this
	}

	if (pcmk_any_flags_set(order->type, pe_order_implies_then
	\|pe_order_implies_first
	\|pe_order_restart)
	&& (first->rsc != NULL)
	&& !pcmk_is_set(first->rsc->flags, pcmk_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pcmk_rsc_blocked)
	&& !pcmk_is_set(first->flags, pe_action_runnable)
	&& pcmk__str_eq(first->task, PCMK_ACTION_STOP, pcmk__str_none)) {

	if (pcmk_is_set(then->flags, pe_action_runnable)) {
	pe__clear_action_flags(then, pe_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pe_rsc_trace(then->rsc, "%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pe_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Update an action's flags for all orderings where it is "then"
	*
	* \param[in,out] then Action to update
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__update_action_for_orderings(pe_action_t then, pe_working_set_t data_set)
	{
	GList *lpc = NULL;
	uint32_t changed = pcmk__updated_none;
	int last_flags = then->flags;

	pe_rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	/* Initialize current known "runnable before" actions. As
	* update_action_for_ordering_flags() is called for each of then's
	* before actions, this number will increment as runnable 'first'
	* actions are encountered.
	*/
	then->runnable_before = 0;

	if (then->required_runnable_before == 0) {
	/* @COMPAT This ordering constraint uses the deprecated
	* "require-all=false" attribute. Treat it like "clone-min=1".
	*/
	then->required_runnable_before = 1;
	}

	/* The pe_order_one_or_more clause of update_action_for_ordering_flags()
	* (called below) will reset runnable if appropriate.
	*/
	pe__clear_action_flags(then, pe_action_runnable);
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;
	pe_action_t *first = other->action;

	pe_node_t *then_node = then->node;
	pe_node_t *first_node = first->node;

	if ((first->rsc != NULL)
	&& (first->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(first->task, PCMK_ACTION_START, pcmk__str_none)) {

	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node != NULL) {
	pe_rsc_trace(first->rsc, "Found %s for 'first' %s",
	pe__node_name(first_node), first->uuid);
	}
	}

	if ((then->rsc != NULL)
	&& (then->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)) {

	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node != NULL) {
	pe_rsc_trace(then->rsc, "Found %s for 'then' %s",
	pe__node_name(then_node), then->uuid);
	}
	}

	// Disable constraint if it only applies when on same node, but isn't
	if (pcmk_is_set(other->type, pe_order_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& !pe__same_node(first_node, then_node)) {

	pe_rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: "
	"not same node",
	other->action->uuid, pe__node_name(first_node),
	then->uuid, pe__node_name(then_node));
	other->type = pe_order_none;
	continue;
	}

	pcmk__clear_updated_flags(changed, then, pcmk__updated_first);

	if ((first->rsc != NULL)
	&& pcmk_is_set(other->type, pe_order_then_cancels_first)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pe__set_action_flags(other->action, pe_action_optional);
	if (!strcmp(first->task, PCMK_ACTION_RELOAD_AGENT)) {
	pe__clear_resource_flags(first->rsc, pcmk_rsc_reload);
	}
	}

	if ((first->rsc != NULL) && (then->rsc != NULL)
	&& (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
	first = action_for_ordering(first);
	}
	if (first != other->action) {
	pe_rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	pe_rsc_trace(then->rsc,
	"%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
	first->uuid, first->flags, then->uuid, then->flags,
	other->type, action_node_str(first));

	if (first == other->action) {
	/* 'first' was not remapped (e.g. from 'start' to 'running'), which
	* could mean it is a non-resource action, a primitive resource
	* action, or already expanded.
	*/
	uint32_t first_flags, then_flags;

	first_flags = action_flags_for_ordering(first, then_node);
	then_flags = action_flags_for_ordering(then, first_node);

	changed \|= update_action_for_ordering_flags(first, then,
	first_flags, then_flags,
	other, data_set);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	other->type = pe_order_none;
	}


	if (pcmk_is_set(changed, pcmk__updated_first)) {
	crm_trace("Re-processing %s and its 'after' actions "
	"because it changed", first->uuid);
	for (GList *lpc2 = first->actions_after; lpc2 != NULL;
	lpc2 = lpc2->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc2->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	pcmk__update_action_for_orderings(first, data_set);
	}
	}

	if (pcmk_is_set(then->flags, pe_action_requires_any)) {
	if (last_flags == then->flags) {
	pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
	} else {
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	}
	}

	if (pcmk_is_set(changed, pcmk__updated_then)) {
	crm_trace("Re-processing %s and its 'after' actions because it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)) {
	pcmk__block_colocation_dependents(then);
	}
	pcmk__update_action_for_orderings(then, data_set);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pe_action_wrapper_t other = (pe_action_wrapper_t ) lpc->data;

	pcmk__update_action_for_orderings(other->action, data_set);
	}
	}
	}

	static inline bool
	is_primitive_action(const pe_action_t *action)
	{
	return (action != NULL) && (action->rsc != NULL)
	&& (action->rsc->variant == pcmk_rsc_variant_primitive);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in,out] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pe__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, false); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Update actions in an asymmetric ordering
	*
	* If the "first" action in an asymmetric ordering is unrunnable, make the
	* "second" action unrunnable as well, if appropriate.
	*
	* \param[in] first 'First' action in an asymmetric ordering
	* \param[in,out] then 'Then' action in an asymmetric ordering
	*/
	static void
	handle_asymmetric_ordering(const pe_action_t first, pe_action_t then)
	{
	/* Only resource actions after an unrunnable 'first' action need updates for
	* asymmetric ordering.
	*/
	if ((then->rsc == NULL) \|\| pcmk_is_set(first->flags, pe_action_runnable)) {
	return;
	}

	// Certain optional 'then' actions are unaffected by unrunnable 'first'
	if (pcmk_is_set(then->flags, pe_action_optional)) {
	enum rsc_role_e then_rsc_role = then->rsc->fns->state(then->rsc, TRUE);

	if ((then_rsc_role == pcmk_role_stopped)
	&& pcmk__str_eq(then->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* If 'then' should stop after 'first' but is already stopped, the
	* ordering is irrelevant.
	*/
	return;
	} else if ((then_rsc_role >= pcmk_role_started)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)
	&& pe__rsc_running_on_only(then->rsc, then->node)) {
	/* Similarly if 'then' should start after 'first' but is already
	* started on a single node.
	*/
	return;
	}
	}

	// 'First' can't run, so 'then' can't either
	clear_action_flag_because(then, pe_action_optional, first);
	clear_action_flag_because(then, pe_action_runnable, first);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in,out] first 'First' action in an ordering with pe_restart_order
	* \param[in,out] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What action flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pe_action_t first, pe_action_t then, uint32_t filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pe_action_runnable)
	&& !pcmk_is_set(then->flags, pe_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pcmk_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pe_rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {
	clear_action_flag_because(then, pe_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (ignored)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pe_action_optional to affect only mandatory
	* actions, and pe_action_runnable to affect only
	* runnable actions)
	* \param[in] type Group of enum pe_ordering flags to apply
	* \param[in,out] data_set Cluster working set
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__update_ordered_actions(pe_action_t first, pe_action_t then,
	const pe_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pe_working_set_t *data_set)
	{
	uint32_t changed = pcmk__updated_none;
	uint32_t then_flags = 0U;
	uint32_t first_flags = 0U;

	CRM_ASSERT((first != NULL) && (then != NULL) && (data_set != NULL));

	then_flags = then->flags;
	first_flags = first->flags;
	if (pcmk_is_set(type, pe_order_asymmetrical)) {
	handle_asymmetric_ordering(first, then);
	}

	if (pcmk_is_set(type, pe_order_implies_first)
	&& !pcmk_is_set(then_flags, pe_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& pcmk_is_set(first_flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}

	if (pcmk_is_set(flags, pe_action_migrate_runnable)
	&& !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable, then);
	}
	}

	if (pcmk_is_set(type, pe_order_promoted_implies_first)
	&& (then->rsc != NULL) && (then->rsc->role == pcmk_role_promoted)
	&& pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(then->flags, pe_action_optional)) {

	clear_action_flag_because(first, pe_action_optional, then);

	if (pcmk_is_set(first->flags, pe_action_migrate_runnable)
	&& !pcmk_is_set(then->flags, pe_action_migrate_runnable)) {
	clear_action_flag_because(first, pe_action_migrate_runnable,
	then);
	}
	}

	if (pcmk_is_set(type, pe_order_implies_first_migratable)
	&& pcmk_is_set(filter, pe_action_optional)) {

	if (!pcmk_all_flags_set(then->flags, pe_action_migrate_runnable
	\|pe_action_runnable)) {
	clear_action_flag_because(first, pe_action_runnable, then);
	}

	if (!pcmk_is_set(then->flags, pe_action_optional)) {
	clear_action_flag_because(first, pe_action_optional, then);
	}
	}

	if (pcmk_is_set(type, pe_order_pseudo_left)
	&& pcmk_is_set(filter, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_runnable)) {

	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	pe__clear_action_flags(then, pe_action_pseudo);
	}

	if (pcmk_is_set(type, pe_order_runnable_left)
	&& pcmk_is_set(filter, pe_action_runnable)
	&& pcmk_is_set(then->flags, pe_action_runnable)
	&& !pcmk_is_set(flags, pe_action_runnable)) {

	clear_action_flag_because(then, pe_action_runnable, first);
	clear_action_flag_because(then, pe_action_migrate_runnable, first);
	}

	if (pcmk_is_set(type, pe_order_implies_then)
	&& pcmk_is_set(filter, pe_action_optional)
	&& pcmk_is_set(then->flags, pe_action_optional)
	&& !pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(first->flags, pe_action_migrate_runnable)) {

	clear_action_flag_because(then, pe_action_optional, first);
	}

	if (pcmk_is_set(type, pe_order_restart)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	pe_rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid, pe__node_name(then->node),
	then->flags, then_flags, first->uuid, first->flags);

	if ((then->rsc != NULL) && (then->rsc->parent != NULL)) {
	// Required to handle "X_stop then X_start" for cloned groups
	pcmk__update_action_for_orderings(then, data_set);
	}
	}

	if (first_flags != first->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	pe_rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid, pe__node_name(first->node),
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Trace-log an action (optionally with its dependent actions)
	*
	* \param[in] pre_text If not NULL, prefix the log with this plus ": "
	* \param[in] action Action to log
	* \param[in] details If true, recursively log dependent actions
	*/
	void
	pcmk__log_action(const char pre_text, const pe_action_t action, bool details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	CRM_CHECK(action != NULL, return);

	if (!pcmk_is_set(action->flags, pe_action_pseudo)) {
	if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	}
	}

	switch (text2task(action->task)) {
	case pcmk_action_fence:
	case pcmk_action_shutdown:
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pe_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else if (pcmk_is_set(action->flags, pe_action_processed)) {
	desc = "";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	const GList *iter = NULL;
	const pe_action_wrapper_t *other = NULL;

	crm_trace("\t\t====== Preceding Actions");
	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	other = (const pe_action_wrapper_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== Subsequent Actions");
	for (iter = action->actions_after; iter != NULL; iter = iter->next) {
	other = (const pe_action_wrapper_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before),
	g_list_length(action->actions_after));
	}
	}

	/*!
	* \internal
	* \brief Create a new shutdown action for a node
	*
	* \param[in,out] node Node being shut down
	*
	* \return Newly created shutdown action for \p node
	*/
	pe_action_t *
	pcmk__new_shutdown_action(pe_node_t *node)
	{
	char *shutdown_id = NULL;
	pe_action_t *shutdown_op = NULL;

	CRM_ASSERT(node != NULL);

	shutdown_id = crm_strdup_printf("%s-%s", PCMK_ACTION_DO_SHUTDOWN,
	node->details->uname);

	shutdown_op = custom_action(NULL, shutdown_id, PCMK_ACTION_DO_SHUTDOWN,
	node, FALSE, TRUE, node->details->data_set);

	pcmk__order_stops_before_shutdown(node, shutdown_op);
	add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	return shutdown_op;
	}

	/*!
	* \internal
	* \brief Calculate and add an operation digest to XML
	*
	* Calculate an operation digest, which enables us to later determine when a
	* restart is needed due to the resource's parameters being changed, and add it
	* to given XML.
	*
	* \param[in] op Operation result from executor
	* \param[in,out] update XML to add digest to
	*/
	static void
	add_op_digest_to_xml(const lrmd_event_data_t op, xmlNode update)
	{
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}
	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, NULL);
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	crm_trace("Creating history XML for %s-interval %s action for %s on %s "
	"(DC version: %s, origin: %s)",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	((node == NULL)? "no node" : node), caller_version, origin);

	task = op->op_type;

	/* Record a successful agent reload as a start, and a failed one as a
	* monitor, to make life easier for the scheduler when determining the
	* current state.
	*
	* @COMPAT We should check "reload" here only if the operation was for a
	* pre-OCF-1.1 resource agent, but we don't know that here, and we should
	* only ever get results for actions scheduled by us, so we can reasonably
	* assume any "reload" is actually a pre-1.1 agent reload.
	*/
	if (pcmk__str_any_of(task, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
	NULL)) {
	if (op->op_status == PCMK_EXEC_DONE) {
	task = PCMK_ACTION_START;
	} else {
	task = PCMK_ACTION_MONITOR;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_EXEC_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	}

	/* Migration history is preserved separately, which usually matters for
	* multiple nodes and is important for future cluster transitions.
	*/
	} else if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	op_id = strdup(key);

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	// Ensure 'last' gets updated, in case record-pending is true
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if (magic == NULL) {
	magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
	(const char *) op->user_data);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, pcmk__s(exit_reason, ""));
	crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); // For context during triage

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	add_op_digest_to_xml(op, xml_op);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	/*!
	* \internal
	* \brief Check whether an action shutdown-locks a resource to a node
	*
	* If the shutdown-lock cluster property is set, resources will not be recovered
	* on a different node if cleanly stopped, and may start only on that same node.
	* This function checks whether that applies to a given action, so that the
	* transition graph can be marked appropriately.
	*
	* \param[in] action Action to check
	*
	* \return true if \p action locks its resource to the action's node,
	* otherwise false
	*/
	bool
	pcmk__action_locks_rsc_to_node(const pe_action_t *action)
	{
	// Only resource actions taking place on resource's lock node are locked
	if ((action == NULL) \|\| (action->rsc == NULL)
	\|\| !pe__same_node(action->node, action->rsc->lock_node)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && (action->task != NULL)
	&& (strcmp(action->task, PCMK_ACTION_STOP) != 0)) {
	return false;
	}

	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pe_action_wrapper_t action_wrapper2 = (const pe_action_wrapper_t )a;
	const pe_action_wrapper_t action_wrapper1 = (const pe_action_wrapper_t )b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}
	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Remove any duplicate action inputs, merging action flags
	*
	* \param[in,out] action Action whose inputs should be checked
	*/
	void
	pcmk__deduplicate_action_inputs(pe_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pe_action_wrapper_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pe_action_wrapper_t input = (pe_action_wrapper_t ) item->data;

	next = item->next;
	if ((last_input != NULL)
	&& (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Output all scheduled actions
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__output_actions(pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;

	// Output node (non-resource) actions
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) iter->data;

	if (action->rsc != NULL) {
	continue; // Resource actions will be output later

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue; // This action was not scheduled
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	task = strdup("Shutdown");

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta,
	"stonith_action");

	task = crm_strdup_printf("Fence (%s)", op);

	} else {
	continue; // Don't display other node action types
	}

	if (pe__is_guest_node(action->node)) {
	const pe_resource_t *remote = action->node->details->remote_rsc;

	node_name = crm_strdup_printf("%s (resource: %s)",
	pe__node_name(action->node),
	remote->container->id);
	} else if (action->node != NULL) {
	node_name = crm_strdup_printf("%s", pe__node_name(action->node));
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}

	// Output resource actions
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->output_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether action from resource history is still in configuration
	*
	* \param[in] rsc Resource that action is for
	* \param[in] task Action's name
	* \param[in] interval_ms Action's interval (in milliseconds)
	*
	* \return true if action is still in resource configuration, otherwise false
	*/
	static bool
	action_in_config(const pe_resource_t rsc, const char task, guint interval_ms)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	bool config = (find_rsc_op_entry(rsc, key) != NULL);

	free(key);
	return config;
	}

	/*!
	* \internal
	* \brief Get action name needed to compare digest for configuration changes
	*
	* \param[in] task Action name from history
	* \param[in] interval_ms Action interval (in milliseconds)
	*
	* \return Action name whose digest should be compared
	*/
	static const char *
	task_for_digest(const char *task, guint interval_ms)
	{
	/* Certain actions need to be compared against the parameters used to start
	* the resource.
	*/
	if ((interval_ms == 0)
	&& pcmk__str_any_of(task, PCMK_ACTION_MONITOR, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_PROMOTE, NULL)) {
	task = PCMK_ACTION_START;
	}
	return task;
	}

	/*!
	* \internal
	* \brief Check whether only sanitized parameters to an action changed
	*
	* When collecting CIB files for troubleshooting, crm_report will mask
	* sensitive resource parameters. If simulations were run using that, affected
	* resources would appear to need a restart, which would complicate
	* troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
	* parameters. This function used that digest to check whether only masked
	* parameters are different.
	*
	* \param[in] xml_op Resource history entry with secure digest
	* \param[in] digest_data Operation digest information being compared
	* \param[in] data_set Cluster working set
	*
	* \return true if only sanitized parameters changed, otherwise false
	*/
	static bool
	only_sanitized_changed(const xmlNode *xml_op,
	const op_digest_cache_t *digest_data,
	const pe_working_set_t *data_set)
	{
	const char *digest_secure = NULL;

	if (!pcmk_is_set(data_set->flags, pcmk_sched_sanitized)) {
	// The scheduler is not being run as a simulation
	return false;
	}

	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);

	return (digest_data->rc != RSC_DIGEST_MATCH) && (digest_secure != NULL)
	&& (digest_data->digest_secure_calc != NULL)
	&& (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
	}

	/*!
	* \internal
	* \brief Force a restart due to a configuration change
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action whose configuration changed
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where resource should be restarted
	*/
	static void
	force_restart(pe_resource_t rsc, const char task, guint interval_ms,
	pe_node_t *node)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	pe_action_t *required = custom_action(rsc, key, task, NULL, FALSE, TRUE,
	rsc->cluster);

	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, node, "Device parameters changed", NULL,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Schedule a reload of a resource on a node
	*
	* \param[in,out] data Resource to reload
	* \param[in] user_data Where resource should be reloaded
	*/
	static void
	schedule_reload(gpointer data, gpointer user_data)
	{
	pe_resource_t *rsc = data;
	const pe_node_t *node = user_data;
	pe_action_t *reload = NULL;

	// For collective resources, just call recursively for children
	if (rsc->variant > pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, schedule_reload, user_data);
	return;
	}

	// Skip the reload in certain situations
	if ((node == NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pe_rsc_trace(rsc, "Skip reload of %s:%s%s %s",
	rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " unmanaged",
	pcmk_is_set(rsc->flags, pcmk_rsc_failed)? " failed" : "",
	(node == NULL)? "inactive" : node->details->uname);
	return;
	}

	/* If a resource's configuration changed while a start was pending,
	* force a full restart instead of a reload.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	pe_rsc_trace(rsc, "%s: preventing agent reload because start pending",
	rsc->id);
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE, TRUE,
	rsc->cluster);
	return;
	}

	// Schedule the reload
	pe__set_resource_flags(rsc, pcmk_rsc_reload);
	reload = custom_action(rsc, reload_key(rsc), PCMK_ACTION_RELOAD_AGENT, node,
	FALSE, TRUE, rsc->cluster);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	// Set orderings so that a required stop or demote cancels the reload
	pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	rsc->cluster);
	pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pe_order_optional\|pe_order_then_cancels_first,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Handle any configuration change for an action
	*
	* Given an action from resource history, if the resource's configuration
	* changed since the action was done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, etc.).
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in,out] node Node that action was on
	* \param[in] xml_op Action XML from resource history
	*
	* \return true if action configuration changed, otherwise false
	*/
	bool
	pcmk__check_action_config(pe_resource_t rsc, pe_node_t node,
	const xmlNode *xml_op)
	{
	guint interval_ms = 0;
	const char *task = NULL;
	const op_digest_cache_t *digest_data = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
	return false);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	CRM_CHECK(task != NULL, return false);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	// If this is a recurring action, check whether it has been orphaned
	if (interval_ms > 0) {
	if (action_in_config(rsc, task, interval_ms)) {
	pe_rsc_trace(rsc, "%s-interval %s for %s on %s is in configuration",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_cancel_removed_actions)) {
	pcmk__schedule_cancel(rsc,
	crm_element_value(xml_op,
	XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "orphan");
	return true;
	} else {
	pe_rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	return true;
	}
	}

	crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node));
	task = task_for_digest(task, interval_ms);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, rsc->cluster);

	if (only_sanitized_changed(xml_op, digest_data, rsc->cluster)) {
	if (!pcmk__is_daemon && (rsc->cluster->priv != NULL)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->info(out,
	"Only 'private' parameters to %s-interval %s for %s "
	"on %s changed: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node),
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}
	return false;
	}

	switch (digest_data->rc) {
	case RSC_DIGEST_RESTART:
	crm_log_xml_debug(digest_data->params_restart, "params:restart");
	force_restart(rsc, task, interval_ms, node);
	return true;

	case RSC_DIGEST_ALL:
	case RSC_DIGEST_UNKNOWN:
	// Changes that can potentially be handled by an agent reload

	if (interval_ms > 0) {
	/* Recurring actions aren't reloaded per se, they are just
	* re-scheduled so the next run uses the new parameters.
	* The old instance will be cancelled automatically.
	*/
	crm_log_xml_debug(digest_data->params_all, "params:reschedule");
	pcmk__reschedule_recurring(rsc, task, interval_ms, node);

	} else if (crm_element_value(xml_op,
	XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
	// Agent supports reload, so use it
	trigger_unfencing(rsc, node,
	"Device parameters changed (reload)", NULL,
	rsc->cluster);
	crm_log_xml_debug(digest_data->params_all, "params:reload");
	schedule_reload((gpointer) rsc, (gpointer) node);

	} else {
	pe_rsc_trace(rsc,
	"Restarting %s "
	"because agent doesn't support reload", rsc->id);
	crm_log_xml_debug(digest_data->params_restart,
	"params:restart");
	force_restart(rsc, task, interval_ms, node);
	}
	return true;

	default:
	break;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create a list of resource's action history entries, sorted by call ID
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[out] start_index Where to store index of start-like action, if any
	* \param[out] stop_index Where to store index of stop action, if any
	*/
	static GList *
	rsc_history_as_list(const xmlNode rsc_entry, int start_index, int *stop_index)
	{
	GList *ops = NULL;

	for (xmlNode *rsc_op = first_named_child(rsc_entry, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
	ops = g_list_prepend(ops, rsc_op);
	}
	ops = g_list_sort(ops, sort_op_by_callid);
	calculate_active_ops(ops, start_index, stop_index);
	return ops;
	}

	/*!
	* \internal
	* \brief Process a resource's action history from the CIB status
	*
	* Given a resource's action history, if the resource's configuration
	* changed since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[in,out] rsc Resource whose history is being processed
	* \param[in,out] node Node whose history is being processed
	*/
	static void
	process_rsc_history(const xmlNode rsc_entry, pe_resource_t rsc,
	pe_node_t *node)
	{
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;
	GList *sorted_op_list = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pe_rsc_is_anon_clone(pe__const_top_resource(rsc, false))) {
	pe_rsc_trace(rsc,
	"Skipping configuration check "
	"for orphaned clone instance %s",
	rsc->id);
	} else {
	pe_rsc_trace(rsc,
	"Skipping configuration check and scheduling clean-up "
	"for orphaned resource %s", rsc->id);
	pcmk__schedule_cleanup(rsc, node, false);
	}
	return;
	}

	if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}
	pe_rsc_trace(rsc,
	"Skipping configuration check for %s "
	"because no longer active on %s",
	rsc->id, pe__node_name(node));
	return;
	}

	pe_rsc_trace(rsc, "Checking for configuration changes for %s on %s",
	rsc->id, pe__node_name(node));

	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}

	sorted_op_list = rsc_history_as_list(rsc_entry, &start_index, &stop_index);
	if (start_index < stop_index) {
	return; // Resource is stopped
	}

	for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	xmlNode rsc_op = (xmlNode ) iter->data;
	const char *task = NULL;
	guint interval_ms = 0;

	if (++offset < start_index) {
	// Skip actions that happened before a start
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0)
	- && (pcmk_is_set(rsc->flags, pe_rsc_maintenance)
	+ && (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)
	\|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	pcmk__schedule_cancel(rsc,
	crm_element_value(rsc_op,
	XML_LRM_ATTR_CALLID),
	task, interval_ms, node, "maintenance mode");

	} else if ((interval_ms > 0)
	\|\| pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't assigned resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pcmk__check_active,
	rsc->cluster);

	} else if (pcmk__check_action_config(rsc, node, rsc_op)
	&& (pe_get_failcount(node, rsc, NULL, pe_fc_effective,
	NULL) != 0)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	rsc->cluster);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	/*!
	* \internal
	* \brief Process a node's action history from the CIB status
	*
	* Given a node's resource history, if the resource's configuration changed
	* since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] node Node whose history is being processed
	* \param[in] lrm_rscs Node's <lrm_resources> from CIB status XML
	*/
	static void
	process_node_history(pe_node_t node, const xmlNode lrm_rscs)
	{
	crm_trace("Processing node history for %s", pe__node_name(node));
	for (const xmlNode *rsc_entry = first_named_child(lrm_rscs,
	XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	if (rsc_entry->children != NULL) {
	GList *result = pcmk__rscs_matching_id(ID(rsc_entry),
	node->details->data_set);

	for (GList *iter = result; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	process_rsc_history(rsc_entry, rsc, node);
	}
	}
	g_list_free(result);
	}
	}
	}

	// XPath to find a node's resource history
	#define XPATH_NODE_HISTORY "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES

	/*!
	* \internal
	* \brief Process any resource configuration changes in the CIB status
	*
	* Go through all nodes' resource history, and if a resource's configuration
	* changed since its actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__handle_rsc_config_changes(pe_working_set_t *data_set)
	{
	crm_trace("Check resource and action configuration for changes");

	/* Rather than iterate through the status section, iterate through the nodes
	* and search for the appropriate status subsection for each. This skips
	* orphaned nodes and lets us eliminate some cases before searching the XML.
	*/
	for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	/* Don't bother checking actions for a node that can't run actions ...
	* unless it's in maintenance mode, in which case we still need to
	* cancel any existing recurring monitors.
	*/
	if (node->details->maintenance
	\|\| pcmk__node_available(node, false, false)) {

	char *xpath = NULL;
	xmlNode *history = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->details->uname);
	history = get_xpath_object(xpath, data_set->input, LOG_NEVER);
	free(xpath);

	process_node_history(node, history);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index 67159a3e99..5830b04c7f 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1489 +1,1489 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(data_set->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	crm_warn("Unknown action '%s' specified in order constraint", action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = char2score(score);

	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pe_warn_once(pe_wo_order_score,
	"Support for 'score' in rsc_order is deprecated "
	"and will be removed in a future release "
	"(use 'kind' instead)");
	}

	} else if (pcmk__str_eq(kind, "Mandatory", pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, "Optional", pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, "Serialize", pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
	"%s to 'Mandatory' because '%s' is not valid",
	pcmk__s(ID(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's symmetrical setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, XML_ORDER_ATTR_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, XML_RULE_ATTR_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit symmetrical setting
	rc = pcmk__xe_get_bool_attr(xml_obj, XML_CONS_ATTR_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
	" for '%s' because not valid with "
	XML_ORDER_ATTR_KIND " of 'Serialize'",
	ID(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pe_order_none; // so we trace-log all flags set

	pe__set_order_flags(flags, pe_order_optional);

	switch (kind) {
	case pe_order_kind_optional:
	break;

	case pe_order_kind_serialize:
	pe__set_order_flags(flags, pe_order_serialize_only);
	break;

	case pe_order_kind_mandatory:
	switch (symmetry) {
	case ordering_asymmetric:
	pe__set_order_flags(flags, pe_order_asymmetrical);
	break;

	case ordering_symmetric:
	pe__set_order_flags(flags, pe_order_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pe__set_order_flags(flags, pe_order_runnable_left);
	}
	break;

	case ordering_symmetric_inverse:
	pe__set_order_flags(flags, pe_order_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] instance_attr XML attribute name for instance number.
	* This option is deprecated and will be removed in a
	* future release.
	* \param[in] data_set Cluster working set
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pe_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const char *instance_attr,
	const pe_working_set_t *data_set)
	{
	// @COMPAT: instance_attr and instance_id variables deprecated since 2.1.5
	pe_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);
	const char *instance_id = crm_element_value(xml, instance_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	ID(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(data_set->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", ID(xml), rsc_id);
	return NULL;
	}

	if (instance_id != NULL) {
	pe_warn_once(pe_wo_order_inst,
	"Support for " XML_ORDER_ATTR_FIRST_INSTANCE " and "
	XML_ORDER_ATTR_THEN_INSTANCE " is deprecated and will be "
	"removed in a future release.");

	if (!pe_rsc_is_clone(rsc)) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"is not a clone but instance '%s' was requested",
	ID(xml), rsc_id, instance_id);
	return NULL;
	}
	rsc = find_clone_instance(rsc, instance_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not have an instance '%s'",
	"'%s'", ID(xml), rsc_id, instance_id);
	return NULL;
	}
	}
	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pe_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pe_rsc_is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INCARNATION_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* require-all=false is deprecated equivalent of clone-min=1
	*/
	if (pcmk__xe_get_bool_attr(xml, "require-all", &require_all) != ENODATA) {
	pe_warn_once(pe_wo_require_all,
	"Support for require-all in ordering constraints "
	"is deprecated and will be removed in a future release"
	" (use clone-min clone meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with clone-min > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pe_action_t *clone_min_met = get_pseudo_op(task, rsc_first->cluster);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;
	pe__set_action_flags(clone_min_met, pe_action_requires_any);

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->children; iter != NULL; iter = iter->next) {
	pe_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pe_order_one_or_more\|pe_order_implies_then_printed,
	rsc_first->cluster);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pe_order_runnable_left, rsc_first->cluster);
	}

	/*!
	* \internal
	* \brief Update ordering flags for restart-type=restart
	*
	* \param[in] rsc 'Then' resource in ordering
	* \param[in] kind Ordering kind
	* \param[in] flag Ordering flag to set (when applicable)
	* \param[in,out] flags Ordering flag set to update
	*
	* \compat The restart-type resource meta-attribute is deprecated. Eventually,
	* it will be removed, and pe_restart_ignore will be the only behavior,
	* at which time this can just be removed entirely.
	*/
	#define handle_restart_type(rsc, kind, flag, flags) do { \
	if (((kind) == pe_order_kind_optional) \
	&& ((rsc)->restart_type == pe_restart_restart)) { \
	pe__set_order_flags((flags), (flag)); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pe_resource_t rsc_first, const char action_first,
	pe_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	handle_restart_type(rsc_then, kind, pe_order_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pe_working_set_t data_set)
	{
	pe_resource_t *rsc_then = NULL;
	pe_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pe_order_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, XML_ORDER_ATTR_FIRST,
	XML_ORDER_ATTR_FIRST_INSTANCE,
	data_set);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, XML_ORDER_ATTR_THEN,
	XML_ORDER_ATTR_THEN_INSTANCE,
	data_set);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	handle_restart_type(rsc_then, kind, pe_order_implies_then, flags);

	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Flag set of enum pe_ordering
	* \param[in,out] sched Cluster working set to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pe_resource_t first_rsc, char first_action_task,
	pe_action_t first_action, pe_resource_t then_rsc,
	char then_action_task, pe_action_t then_action,
	uint32_t flags, pe_working_set_t *sched)
	{
	pe__ordering_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = calloc(1, sizeof(pe__ordering_t));
	CRM_ASSERT(order != NULL);

	order->id = sched->order_id++;
	order->flags = flags;
	order->lh_rsc = first_rsc;
	order->rh_rsc = then_rsc;
	order->lh_action = first_action;
	order->rh_action = then_action;
	order->lh_action_task = first_action_task;
	order->rh_action_task = then_action_task;

	if ((order->lh_action_task == NULL) && (first_action != NULL)) {
	order->lh_action_task = strdup(first_action->uuid);
	}

	if ((order->rh_action_task == NULL) && (then_action != NULL)) {
	order->rh_action_task = strdup(then_action->uuid);
	}

	if ((order->lh_rsc == NULL) && (first_action != NULL)) {
	order->lh_rsc = first_action->rsc;
	}

	if ((order->rh_rsc == NULL) && (then_action != NULL)) {
	order->rh_rsc = then_action->rsc;
	}

	pe_rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->order_id - 1),
	pcmk__s(order->lh_action_task, "an underspecified action"),
	pcmk__s(order->rh_action_task, "an underspecified action"));

	sched->ordering_constraints = g_list_prepend(sched->ordering_constraints,
	order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind rsc_order XML "kind" attribute
	* \param[in] parent_symmetrical_s rsc_order XML "symmetrical" attribute
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pe_working_set_t data_set)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pe_resource_t *last = NULL;
	pe_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pe_order_optional;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = ID(set);
	const char *action = crm_element_value(set, "action");
	const char *sequential_s = crm_element_value(set, "sequential");
	const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pe_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, data_set);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pe_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] data_set Cluster working set
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pe_working_set_t *data_set,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pe_resource_t *rsc_1 = NULL;
	pe_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, "action");
	const char *action_2 = crm_element_value(set2, "action");

	uint32_t flags = pe_order_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, "require-all", &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s", ID(set1));
	pe_action_t *unordered_action = get_pseudo_op(task, data_set);

	free(task);
	pe__set_action_flags(unordered_action, pe_action_requires_any);

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pe_order_one_or_more
	\|pe_order_implies_then_printed,
	data_set);
	}
	for (xml_rsc_2 = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pe_order_runnable_left, data_set);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, "sequential")) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	rid = ID(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = first_named_child(set2, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = first_named_child(set1, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));

	for (xmlNode *xml_rsc_2 = first_named_child(set2,
	XML_TAG_RESOURCE_REF);
	xml_rsc_2 != NULL; xml_rsc_2 = crm_next_same_xml(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] data_set Cluster working set
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pe_working_set_t *data_set)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pe_resource_t *rsc_first = NULL;
	pe_resource_t *rsc_then = NULL;
	pe_tag_t *tag_first = NULL;
	pe_tag_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, data_set);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
	id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", ID(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);

	*expanded_xml = copy_xml(xml_obj);

	// Convert template/tag reference in "first" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	// Move "first-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_first, "action", action_first);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_FIRST_ACTION);
	}
	any_sets = true;
	}

	// Convert template/tag reference in "then" into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, XML_ORDER_ATTR_THEN,
	true, data_set)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	// Move "then-action" into converted resource_set as "action"
	crm_xml_add(rsc_set_then, "action", action_then);
	xml_remove_prop(*expanded_xml, XML_ORDER_ATTR_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_order");
	} else {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, data_set) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	set = expand_idref(set, data_set->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, data_set) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, data_set,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, data_set,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	free_xml(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, data_set);
	}
	}

	static bool
	ordering_is_invalid(pe_action_t action, pe_action_wrapper_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->type, pe_order_preserve)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	crm_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->type == pe_order_load) && action->rsc
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pe_working_set_t *data_set)
	{
	for (GList *iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->type = pe_order_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pe_node_t node, pe_action_t shutdown_op)
	{
	for (GList *iter = node->details->data_set->actions;
	iter != NULL; iter = iter->next) {

	pe_action_t action = (pe_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if (!pe__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	- if (pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)) {
	+ if (pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pe__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk_rsc_managed\|pcmk_rsc_blocked)) {
	pe_rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pe__node_name(node));
	continue;
	}

	pe_rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pe_action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pe_order_optional\|pe_order_runnable_left,
	node->details->data_set);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pe_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->actions, key, NULL);
	free(key);
	free(task);
	} else {
	crm_err("Invalid operation key (bug?): %s", original_key);
	}
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pe_action_t *first_action,
	const pe_resource_t rsc, pe__ordering_t order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pe_order_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pe_rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->rh_action != NULL) {
	then_actions = g_list_prepend(NULL, order->rh_action);

	} else {
	then_actions = find_actions_by_task(rsc, order->rh_action_task);
	}

	if (then_actions == NULL) {
	pe_rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->rh_action_task, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pe_action_dangle)) {

	pe_rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->rh_action_task, rsc->id);
	pe__clear_order_flags(flags, pe_order_implies_then);
	}

	if ((first_action == NULL) && !pcmk_is_set(flags, pe_order_implies_then)) {
	pe_rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pe_action_t then_action_iter = (pe_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pe__clear_action_flags(then_action_iter, pe_action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->lh_action_task, order->lh_rsc->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pe_resource_t first_rsc, pe__ordering_t order)
	{
	GList *first_actions = NULL;
	pe_action_t *first_action = order->lh_action;
	pe_resource_t *then_rsc = order->rh_rsc;

	CRM_ASSERT(first_rsc != NULL);
	pe_rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->lh_action_task);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	if ((first_rsc->fns->state(first_rsc, TRUE) == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else if ((first_rsc->fns->state(first_rsc,
	TRUE) == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pe_rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->lh_action_task, first_rsc->id);

	} else {
	pe_rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->lh_action_task, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	TRUE, first_rsc->cluster);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->rh_action == NULL) {
	pe_rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->rh_action->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->rh_action, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pe_action_t *) data,
	(pe_working_set_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Cluster working set
	*/
	void
	pcmk__apply_orderings(pe_working_set_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->ordering_constraints = g_list_reverse(sched->ordering_constraints);

	for (GList *iter = sched->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pe__ordering_t *order = iter->data;
	pe_resource_t *rsc = order->lh_rsc;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	order_resource_actions_after(order->lh_action, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->lh_action, order->rh_action, order->flags);
	}
	}

	g_list_foreach(sched->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->actions));
	g_list_foreach(sched->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pe_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pe_action_t before = (pe_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pe__node_name(before->node),
	after_desc, pe__node_name(after->node));
	order_actions(before, after, pe_order_optional);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pe_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pe_order_optional);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pe_order_optional);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pe_order_optional);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pe_order_optional);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pe_order_optional);
	}
	diff --git a/lib/pacemaker/pcmk_sched_recurring.c b/lib/pacemaker/pcmk_sched_recurring.c
	index 0725f2564a..a8260fd141 100644
	--- a/lib/pacemaker/pcmk_sched_recurring.c
	+++ b/lib/pacemaker/pcmk_sched_recurring.c
	@@ -1,718 +1,718 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Information parsed from an operation history entry in the CIB
	struct op_history {
	// XML attributes
	const char *id; // ID of history entry
	const char *name; // Action name

	// Parsed information
	char *key; // Operation key for action
	enum rsc_role_e role; // Action role (or pcmk_role_unknown for default)
	guint interval_ms; // Action interval
	};

	/*!
	* \internal
	* \brief Parse an interval from XML
	*
	* \param[in] xml XML containing an interval attribute
	*
	* \return Interval parsed from XML (or 0 as default)
	*/
	static guint
	xe_interval(const xmlNode *xml)
	{
	return crm_parse_interval_spec(crm_element_value(xml,
	XML_LRM_ATTR_INTERVAL));
	}

	/*!
	* \internal
	* \brief Check whether an operation exists multiple times in resource history
	*
	* \param[in] rsc Resource with history to search
	* \param[in] name Name of action to search for
	* \param[in] interval_ms Interval (in milliseconds) of action to search for
	*
	* \return true if an operation with \p name and \p interval_ms exists more than
	* once in the operation history of \p rsc, otherwise false
	*/
	static bool
	is_op_dup(const pe_resource_t rsc, const char name, guint interval_ms)
	{
	const char *id = NULL;

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	// Check whether action name and interval match
	if (!pcmk__str_eq(crm_element_value(op, "name"), name, pcmk__str_none)
	\|\| (xe_interval(op) != interval_ms)) {
	continue;
	}

	if (ID(op) == NULL) {
	continue; // Shouldn't be possible
	}

	if (id == NULL) {
	id = ID(op); // First matching op
	} else {
	pcmk__config_err("Operation %s is duplicate of %s (do not use "
	"same name and interval combination more "
	"than once per resource)", ID(op), id);
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether an action name is one that can be recurring
	*
	* \param[in] name Action name to check
	*
	* \return true if \p name is an action known to be unsuitable as a recurring
	* operation, otherwise false
	*
	* \note Pacemaker's current philosophy is to allow users to configure recurring
	* operations except for a short list of actions known not to be suitable
	* for that (as opposed to allowing only actions known to be suitable,
	* which includes only monitor). Among other things, this approach allows
	* users to define their own custom operations and make them recurring,
	* though that use case is not well tested.
	*/
	static bool
	op_cannot_recur(const char *name)
	{
	return pcmk__str_any_of(name, PCMK_ACTION_STOP, PCMK_ACTION_START,
	PCMK_ACTION_DEMOTE, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_RELOAD_AGENT,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL);
	}

	/*!
	* \internal
	* \brief Check whether a resource history entry is for a recurring action
	*
	* \param[in] rsc Resource that history entry is for
	* \param[in] xml XML of resource history entry to check
	* \param[out] op Where to store parsed info if recurring
	*
	* \return true if \p xml is for a recurring action, otherwise false
	*/
	static bool
	is_recurring_history(const pe_resource_t rsc, const xmlNode xml,
	struct op_history *op)
	{
	const char *role = NULL;

	op->interval_ms = xe_interval(xml);
	if (op->interval_ms == 0) {
	return false; // Not recurring
	}

	op->id = ID(xml);
	if (pcmk__str_empty(op->id)) {
	pcmk__config_err("Ignoring resource history entry without ID");
	return false; // Shouldn't be possible (unless CIB was manually edited)
	}

	op->name = crm_element_value(xml, "name");
	if (op_cannot_recur(op->name)) {
	pcmk__config_err("Ignoring %s because %s action cannot be recurring",
	op->id, pcmk__s(op->name, "unnamed"));
	return false;
	}

	// There should only be one recurring operation per action/interval
	if (is_op_dup(rsc, op->name, op->interval_ms)) {
	return false;
	}

	// Ensure role is valid if specified
	role = crm_element_value(xml, "role");
	if (role == NULL) {
	op->role = pcmk_role_unknown;
	} else {
	op->role = text2role(role);
	if (op->role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring %s because %s is not a valid role",
	op->id, role);
	}
	}

	// Disabled resources don't get monitored
	op->key = pcmk__op_key(rsc->id, op->name, op->interval_ms);
	if (find_rsc_op_entry(rsc, op->key) == NULL) {
	crm_trace("Not creating recurring action %s for disabled resource %s",
	op->id, rsc->id);
	free(op->key);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a recurring action for an active role should be optional
	*
	* \param[in] rsc Resource that recurring action is for
	* \param[in] node Node that \p rsc will be active on (if any)
	* \param[in] key Operation key for recurring action to check
	* \param[in,out] start Start action for \p rsc
	*
	* \return true if recurring action should be optional, otherwise false
	*/
	static bool
	active_recurring_should_be_optional(const pe_resource_t *rsc,
	const pe_node_t node, const char key,
	pe_action_t *start)
	{
	GList *possible_matches = NULL;

	if (node == NULL) { // Should only be possible if unmanaged and stopped
	pe_rsc_trace(rsc, "%s will be mandatory because resource is unmanaged",
	key);
	return false;
	}

	if (!pcmk_is_set(rsc->cmds->action_flags(start, NULL),
	pe_action_optional)) {
	pe_rsc_trace(rsc, "%s will be mandatory because %s is",
	key, start->uuid);
	return false;
	}

	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "%s will be mandatory because it is not active on %s",
	key, pe__node_name(node));
	return false;
	}

	for (const GList *iter = possible_matches;
	iter != NULL; iter = iter->next) {

	const pe_action_t op = (const pe_action_t ) iter->data;

	if (pcmk_is_set(op->flags, pe_action_reschedule)) {
	pe_rsc_trace(rsc,
	"%s will be mandatory because "
	"it needs to be rescheduled", key);
	g_list_free(possible_matches);
	return false;
	}
	}

	g_list_free(possible_matches);
	return true;
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for an active role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in,out] start Start action for \p rsc on \p node
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_active(pe_resource_t rsc, pe_action_t start,
	const pe_node_t node, const struct op_history op)
	{
	pe_action_t *mon = NULL;
	bool is_optional = true;
	const bool is_default_role = (op->role == pcmk_role_unknown);

	// We're only interested in recurring actions for active roles
	if (op->role == pcmk_role_stopped) {
	return;
	}

	is_optional = active_recurring_should_be_optional(rsc, node, op->key,
	start);

	if ((!is_default_role && (rsc->next_role != op->role))
	\|\| (is_default_role && (rsc->next_role == pcmk_role_promoted))) {
	// Configured monitor role doesn't match role resource will have

	if (is_optional) { // It's running, so cancel it
	char *after_key = NULL;
	pe_action_t *cancel_op = pcmk__new_cancel_action(rsc, op->name,
	op->interval_ms,
	node);

	switch (rsc->role) {
	case pcmk_role_unpromoted:
	case pcmk_role_started:
	if (rsc->next_role == pcmk_role_promoted) {
	after_key = promote_key(rsc);

	} else if (rsc->next_role == pcmk_role_stopped) {
	after_key = stop_key(rsc);
	}

	break;
	case pcmk_role_promoted:
	after_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (after_key) {
	pcmk__new_ordering(rsc, NULL, cancel_op, rsc, after_key, NULL,
	pe_order_runnable_left, rsc->cluster);
	}
	}

	do_crm_log((is_optional? LOG_INFO : LOG_TRACE),
	"%s recurring action %s because %s configured for %s role "
	"(not %s)",
	(is_optional? "Cancelling" : "Ignoring"), op->key, op->id,
	role2text(is_default_role? pcmk_role_unpromoted : op->role),
	role2text(rsc->next_role));
	return;
	}

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s %s on %s)",
	(is_optional? "optional" : "mandatory"), op->key,
	op->id, rsc->id, role2text(rsc->next_role),
	pe__node_name(node));

	mon = custom_action(rsc, strdup(op->key), op->name, node, is_optional, TRUE,
	rsc->cluster);

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	pe_rsc_trace(rsc, "%s is unrunnable because start is", mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if ((node == NULL) \|\| !node->details->online
	\|\| node->details->unclean) {
	pe_rsc_trace(rsc, "%s is unrunnable because no node is available",
	mon->uuid);
	pe__clear_action_flags(mon, pe_action_runnable);

	} else if (!pcmk_is_set(mon->flags, pe_action_optional)) {
	pe_rsc_info(rsc, "Start %s-interval %s for %s on %s",
	pcmk__readable_interval(op->interval_ms), mon->task,
	rsc->id, pe__node_name(node));
	}

	if (rsc->next_role == pcmk_role_promoted) {
	pe__add_action_expected_result(mon, CRM_EX_PROMOTED);
	}

	// Order monitor relative to other actions
	if ((node == NULL) \|\| pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__new_ordering(rsc, start_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	pcmk__new_ordering(rsc, reload_key(rsc), NULL,
	NULL, strdup(mon->uuid), mon,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	if (rsc->next_role == pcmk_role_promoted) {
	pcmk__new_ordering(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	rsc->cluster);

	} else if (rsc->role == pcmk_role_promoted) {
	pcmk__new_ordering(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional\|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief Cancel a recurring action if running on a node
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node to cancel action on
	* \param[in] key Operation key for action
	* \param[in] name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	*/
	static void
	cancel_if_running(pe_resource_t rsc, const pe_node_t node, const char *key,
	const char *name, guint interval_ms)
	{
	GList *possible_matches = find_actions_exact(rsc->actions, key, node);
	pe_action_t *cancel_op = NULL;

	if (possible_matches == NULL) {
	return; // Recurring action isn't running on this node
	}
	g_list_free(possible_matches);

	cancel_op = pcmk__new_cancel_action(rsc, name, interval_ms, node);

	switch (rsc->next_role) {
	case pcmk_role_started:
	case pcmk_role_unpromoted:
	/* Order starts after cancel. If the current role is
	* stopped, this cancels the monitor before the resource
	* starts; if the current role is started, then this cancels
	* the monitor on a migration target before starting there.
	*/
	pcmk__new_ordering(rsc, NULL, cancel_op,
	rsc, start_key(rsc), NULL,
	pe_order_runnable_left, rsc->cluster);
	break;
	default:
	break;
	}
	pe_rsc_info(rsc,
	"Cancelling %s-interval %s action for %s on %s because "
	"configured for " PCMK__ROLE_STOPPED " role (not %s)",
	pcmk__readable_interval(interval_ms), name, rsc->id,
	pe__node_name(node), role2text(rsc->next_role));
	}

	/*!
	* \internal
	* \brief Order an action after all probes of a resource on a node
	*
	* \param[in,out] rsc Resource to check for probes
	* \param[in] node Node to check for probes of \p rsc
	* \param[in,out] action Action to order after probes of \p rsc on \p node
	*/
	static void
	order_after_probes(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *probes = pe__resource_actions(rsc, node, PCMK_ACTION_MONITOR, FALSE);

	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	order_actions((pe_action_t *) iter->data, action,
	pe_order_runnable_left);
	}
	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Order an action after all stops of a resource on a node
	*
	* \param[in,out] rsc Resource to check for stops
	* \param[in] node Node to check for stops of \p rsc
	* \param[in,out] action Action to order after stops of \p rsc on \p node
	*/
	static void
	order_after_stops(pe_resource_t rsc, const pe_node_t node,
	pe_action_t *action)
	{
	GList *stop_ops = pe__resource_actions(rsc, node, PCMK_ACTION_STOP, TRUE);

	for (GList *iter = stop_ops; iter != NULL; iter = iter->next) {
	pe_action_t stop = (pe_action_t ) iter->data;

	if (!pcmk_is_set(stop->flags, pe_action_optional)
	&& !pcmk_is_set(action->flags, pe_action_optional)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc, "%s optional on %s: unmanaged",
	action->uuid, pe__node_name(node));
	pe__set_action_flags(action, pe_action_optional);
	}

	if (!pcmk_is_set(stop->flags, pe_action_runnable)) {
	crm_debug("%s unrunnable on %s: stop is unrunnable",
	action->uuid, pe__node_name(node));
	pe__clear_action_flags(action, pe_action_runnable);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__new_ordering(rsc, stop_key(rsc), stop,
	NULL, NULL, action,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);
	}
	}
	g_list_free(stop_ops);
	}

	/*!
	* \internal
	* \brief Create recurring action from resource history entry for inactive role
	*
	* \param[in,out] rsc Resource that resource history is for
	* \param[in] node Node that resource will be active on (if any)
	* \param[in] op Resource history entry
	*/
	static void
	recurring_op_for_inactive(pe_resource_t rsc, const pe_node_t node,
	const struct op_history *op)
	{
	GList *possible_matches = NULL;

	// We're only interested in recurring actions for the inactive role
	if (op->role != pcmk_role_stopped) {
	return;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	crm_notice("Ignoring %s (recurring monitors for " PCMK__ROLE_STOPPED
	" role are not supported for anonymous clones)", op->id);
	return; // @TODO add support
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on nodes "
	"where it should not be running", op->id, rsc->id);

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pe_node_t stop_node = (pe_node_t ) iter->data;

	bool is_optional = true;
	pe_action_t *stopped_mon = NULL;

	// Cancel action on node where resource will be active
	if ((node != NULL)
	&& pcmk__str_eq(stop_node->details->uname, node->details->uname,
	pcmk__str_casei)) {
	cancel_if_running(rsc, node, op->key, op->name, op->interval_ms);
	continue;
	}

	// Recurring action on this node is optional if it's already active here
	possible_matches = find_actions_exact(rsc->actions, op->key, stop_node);
	is_optional = (possible_matches != NULL);
	g_list_free(possible_matches);

	pe_rsc_trace(rsc,
	"Creating %s recurring action %s for %s (%s "
	PCMK__ROLE_STOPPED " on %s)",
	(is_optional? "optional" : "mandatory"),
	op->key, op->id, rsc->id, pe__node_name(stop_node));

	stopped_mon = custom_action(rsc, strdup(op->key), op->name, stop_node,
	is_optional, TRUE, rsc->cluster);

	pe__add_action_expected_result(stopped_mon, CRM_EX_NOT_RUNNING);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	order_after_probes(rsc, stop_node, stopped_mon);
	}

	/* The recurring action is for the inactive role, so it shouldn't be
	* performed until the resource is inactive.
	*/
	order_after_stops(rsc, stop_node, stopped_mon);

	if (!stop_node->details->online \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s unrunnable on %s: node unavailable)",
	stopped_mon->uuid, pe__node_name(stop_node));
	pe__clear_action_flags(stopped_mon, pe_action_runnable);
	}

	if (pcmk_is_set(stopped_mon->flags, pe_action_runnable)
	&& !pcmk_is_set(stopped_mon->flags, pe_action_optional)) {
	crm_notice("Start recurring %s-interval %s for "
	PCMK__ROLE_STOPPED " %s on %s",
	pcmk__readable_interval(op->interval_ms),
	stopped_mon->task, rsc->id, pe__node_name(stop_node));
	}
	}
	}

	/*!
	* \internal
	* \brief Create recurring actions for a resource
	*
	* \param[in,out] rsc Resource to create recurring actions for
	*/
	void
	pcmk__create_recurring_actions(pe_resource_t *rsc)
	{
	pe_action_t *start = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for blocked resource %s",
	rsc->id);
	return;
	}

	- if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	+ if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pe_rsc_trace(rsc, "Skipping recurring actions for %s "
	"in maintenance mode", rsc->id);
	return;
	}

	if (rsc->allocated_to == NULL) {
	// Recurring actions for active roles not needed

	} else if (rsc->allocated_to->details->maintenance) {
	pe_rsc_trace(rsc,
	"Skipping recurring actions for %s on %s "
	"in maintenance mode",
	rsc->id, pe__node_name(rsc->allocated_to));

	} else if ((rsc->next_role != pcmk_role_stopped)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	// Recurring actions for active roles needed
	start = start_action(rsc, rsc->allocated_to, TRUE);
	}

	pe_rsc_trace(rsc, "Creating any recurring actions needed for %s", rsc->id);

	for (xmlNode *op = first_named_child(rsc->ops_xml, "op");
	op != NULL; op = crm_next_same_xml(op)) {

	struct op_history op_history = { NULL, };

	if (!is_recurring_history(rsc, op, &op_history)) {
	continue;
	}

	if (start != NULL) {
	recurring_op_for_active(rsc, start, rsc->allocated_to, &op_history);
	}
	recurring_op_for_inactive(rsc, rsc->allocated_to, &op_history);

	free(op_history.key);
	}
	}

	/*!
	* \internal
	* \brief Create an executor cancel action
	*
	* \param[in,out] rsc Resource of action to cancel
	* \param[in] task Name of action to cancel
	* \param[in] interval_ms Interval of action to cancel
	* \param[in] node Node of action to cancel
	*
	* \return Created op
	*/
	pe_action_t *
	pcmk__new_cancel_action(pe_resource_t rsc, const char task, guint interval_ms,
	const pe_node_t *node)
	{
	pe_action_t *cancel_op = NULL;
	char *key = NULL;
	char *interval_ms_s = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL) && (node != NULL));

	// @TODO dangerous if possible to schedule another action with this key
	key = pcmk__op_key(rsc->id, task, interval_ms);

	cancel_op = custom_action(rsc, key, PCMK_ACTION_CANCEL, node, FALSE, TRUE,
	rsc->cluster);

	pcmk__str_update(&cancel_op->task, PCMK_ACTION_CANCEL);
	pcmk__str_update(&cancel_op->cancel_task, task);

	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
	free(interval_ms_s);

	return cancel_op;
	}

	/*!
	* \internal
	* \brief Schedule cancellation of a recurring action
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] call_id Action's call ID from history
	* \param[in] task Action name
	* \param[in] interval_ms Action interval
	* \param[in] node Node that history entry is for
	* \param[in] reason Short description of why action is cancelled
	*/
	void
	pcmk__schedule_cancel(pe_resource_t rsc, const char call_id, const char *task,
	guint interval_ms, const pe_node_t *node,
	const char *reason)
	{
	pe_action_t *cancel = NULL;

	CRM_CHECK((rsc != NULL) && (task != NULL)
	&& (node != NULL) && (reason != NULL),
	return);

	crm_info("Recurring %s-interval %s for %s will be stopped on %s: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pe__node_name(node), reason);
	cancel = pcmk__new_cancel_action(rsc, task, interval_ms, node);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);

	// Cancellations happen after stops
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, rsc, NULL, cancel,
	pe_order_optional, rsc->cluster);
	}

	/*!
	* \internal
	* \brief Reschedule a recurring action
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action being rescheduled
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where action should be rescheduled
	*/
	void
	pcmk__reschedule_recurring(pe_resource_t rsc, const char task,
	guint interval_ms, pe_node_t *node)
	{
	pe_action_t *op = NULL;

	trigger_unfencing(rsc, node, "Device parameters changed (reschedule)",
	NULL, rsc->cluster);
	op = custom_action(rsc, pcmk__op_key(rsc->id, task, interval_ms),
	task, node, TRUE, TRUE, rsc->cluster);
	pe__set_action_flags(op, pe_action_reschedule);
	}

	/*!
	* \internal
	* \brief Check whether an action is recurring
	*
	* \param[in] action Action to check
	*
	* \return true if \p action has a nonzero interval, otherwise false
	*/
	bool
	pcmk__action_is_recurring(const pe_action_t *action)
	{
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return false;
	}
	return (interval_ms > 0);
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 267f25955c..e0e1993de1 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2217 +1,2218 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pe_resource_t *child;

	GList replicas; // pe__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pe_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pe_resource_t *
	pe__bundled_resource(const pe_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pe_resource_t *
	pe__get_rsc_in_container(const pe_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pe_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pe__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pe_resource_t bundle, const pe_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pe__bundle_replica_t *replica = iter->data;

	if (pe__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pe_resource_t *
	pe__first_container(const pe_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pe__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pe_resource_t *bundle,
	bool (fn)(pe__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pe__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pe_resource_t *bundle,
	bool (fn)(const pe__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pe__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pe__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	crm_xml_add(rsc, XML_ATTR_ID, name);
	crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
	crm_xml_add(rsc, XML_ATTR_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pe_resource_t parent, pe__bundle_variant_data_t data,
	pe__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = create_xml_node(xml_ip, "operations");
	crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pe_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pe__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_remote_port=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_remote_port=%d", env_opt,
	DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network, "host",
	pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = create_xml_node(xml_container, "operations");
	crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pe__set_resource_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pe_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pe_node_t *) match)->weight = -INFINITY;
	((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pe_resource_t parent, pe__bundle_variant_data_t data,
	pe__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pe_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during data set cleanup to use as
	* the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = ID(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, "remote", "-INFINITY",
	parent->cluster);
	} else {
	node->weight = -INFINITY;
	}
	node->rsc_discover_mode = pe_discover_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
	* Unfortunately, a bundle has to be mostly unpacked before it's obvious
	* what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pe_discover_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pe_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pe__is_guest_or_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pe__is_guest_node() functions correctly immediately
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	g_hash_table_insert(replica->node->details->attrs,
	strdup(CRM_ATTR_KIND), strdup("container"));

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pe_resource_t parent, pe__bundle_variant_data_t data,
	pe__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pe__set_resource_flags(replica->remote,
	pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));

	CRM_ASSERT(mount != NULL);
	mount->source = strdup(source);
	mount->target = strdup(target);
	pcmk__str_update(&mount->options, options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pe__bundle_replica_t *
	replica_for_remote(pe_resource_t *remote)
	{
	pe_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pe_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pe_resource_t rsc, pe_working_set_t data_set,
	xmlNode xml, const char field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pe_node_t *node = NULL;
	pe__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pe__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pe__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", ID(mount_xml), flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid promoted-max
	value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, "masters");
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	// Default replicas to promoted-max if it was specified and 1 otherwise
	value = crm_element_value(xml_obj, "replicas");
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, "replicas-per-host");
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
	bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
	bundle_data->image = crm_element_value_copy(xml_obj, "image");
	bundle_data->container_network = crm_element_value_copy(xml_obj, "network");

	xml_obj = first_named_child(rsc->xml, "network");
	if(xml_obj) {

	bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
	bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
	bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
	bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
	value = crm_element_value(xml_obj, "add-host");
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
	port->source = crm_element_value_copy(xml_child, "port");

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, "range");
	} else {
	port->target = crm_element_value_copy(xml_child, "internal-port");
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pe_err("Invalid port directive %s", ID(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = first_named_child(rsc->xml, "storage");
	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	const char *source = crm_element_value(xml_child, "source-dir");
	const char *target = crm_element_value(xml_child, "target-dir");
	const char *options = crm_element_value(xml_child, "options");
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, "source-dir-root");
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pe_err("Invalid mount directive %s", ID(xml_child));
	}
	}

	xml_obj = first_named_child(rsc->xml, "primitive");
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_INCARNATION_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_INCARNATION_NODEMAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
	add_node_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
	rsc->id, ID(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	data_set) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = calloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting control-port.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));

	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pe__set_resource_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
	XML_RSC_ATTR_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	g_hash_table_replace(bundle_data->child->meta,
	strdup(XML_RSC_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));

	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pe_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pe_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pe_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pe__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pe_resource_t *
	pe__find_bundle_replica(const pe_resource_t bundle, const pe_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (replica->node->details == node->details) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " XML_ATTR_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
	"id", rsc->id,
	"type", container_agent_str(bundle_data->agent_type),
	"image", bundle_data->image,
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	- "maintenance", pe__rsc_bool_str(rsc, pe_rsc_maintenance),
	+ "maintenance",
	+ pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"description", desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t out, pe__bundle_replica_t replica,
	pe_node_t *node, uint32_t show_opts)
	{
	pe_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pe_resource_t *rsc)
	{
	- if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	+ if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t out, pe__bundle_replica_t replica,
	pe_node_t *node, uint32_t show_opts)
	{
	const pe_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pe__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pe_node_t *node = NULL;
	pe_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pe__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pe__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pe_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pe_rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pe_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pe_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pe__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pe_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pe__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pe_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pe__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of resource_object_functions_t:active_node()
	pe_node_t *
	pe__bundle_active_node(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pe_node_t *active = NULL;
	pe_node_t *node = NULL;
	pe_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pe__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pe_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
	index 9c05c21429..9e9dc31098 100644
	--- a/lib/pengine/clone.c
	+++ b/lib/pengine/clone.c
	@@ -1,1523 +1,1524 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED_LEGACY "s"
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED_LEGACY "s"
	#else
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED
	#endif

	typedef struct clone_variant_data_s {
	int clone_max;
	int clone_node_max;

	int promoted_max;
	int promoted_node_max;

	int total_clones;

	uint32_t flags; // Group of enum pe__clone_flags

	notify_data_t *stop_notify;
	notify_data_t *start_notify;
	notify_data_t *demote_notify;
	notify_data_t *promote_notify;

	xmlNode *xml_obj_child;
	} clone_variant_data_t;

	#define get_clone_variant_data(data, rsc) \
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_clone)); \
	data = (clone_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be run
	*
	* \param[in] clone Clone or clone instance to check
	*
	* \return Maximum instances for \p clone
	*/
	int
	pe__clone_max(const pe_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed per node
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum allowed instances per node for \p clone
	*/
	int
	pe__clone_node_max(const pe_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_node_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_max(const pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_node_max(const pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_node_max;
	}

	static GList *
	sorted_hash_table_values(GHashTable *table)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
	retval = g_list_prepend(retval, (char *) value);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) strcmp);
	return retval;
	}

	static GList *
	nodes_with_status(GHashTable table, const char status)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!strcmp((char *) value, status)) {
	retval = g_list_prepend(retval, key);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
	return retval;
	}

	static GString *
	node_list_to_str(const GList *list)
	{
	GString *retval = NULL;

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__add_word(&retval, 1024, (const char *) iter->data);
	}

	return retval;
	}

	static void
	clone_header(pcmk__output_t out, int rc, const pe_resource_t *rsc,
	clone_variant_data_t clone_data, const char desc)
	{
	GString *attrs = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__add_separated_word(&attrs, 64, "promotable", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	pcmk__add_separated_word(&attrs, 64, "unique", ", ");
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
	}

	- if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	+ if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
	}

	if (attrs != NULL) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s] (%s)%s%s%s",
	rsc->id, ID(clone_data->xml_obj_child),
	(const char *) attrs->str, desc ? " (" : "",
	desc ? desc : "", desc ? ")" : "");
	g_string_free(attrs, TRUE);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s]%s%s%s",
	rsc->id, ID(clone_data->xml_obj_child),
	desc ? " (" : "", desc ? desc : "",
	desc ? ")" : "");
	}
	}

	void
	pe__force_anon(const char standard, pe_resource_t rsc, const char *rid,
	pe_working_set_t *data_set)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	pe_warn("Ignoring " XML_RSC_ATTR_UNIQUE " for %s because %s resources "
	"such as %s can be used only as anonymous clones",
	rsc->id, standard, rid);

	clone_data->clone_node_max = 1;
	clone_data->clone_max = QB_MIN(clone_data->clone_max,
	g_list_length(data_set->nodes));
	}
	}

	pe_resource_t *
	find_clone_instance(const pe_resource_t rsc, const char sub_id)
	{
	char *child_id = NULL;
	pe_resource_t *child = NULL;
	const char *child_base = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	child_base = ID(clone_data->xml_obj_child);
	child_id = crm_strdup_printf("%s:%s", child_base, sub_id);
	child = pe_find_resource(rsc->children, child_id);

	free(child_id);
	return child;
	}

	pe_resource_t *
	pe__create_clone_child(pe_resource_t rsc, pe_working_set_t data_set)
	{
	gboolean as_orphan = FALSE;
	char *inc_num = NULL;
	char *inc_max = NULL;
	pe_resource_t *child_rsc = NULL;
	xmlNode *child_copy = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);

	if (clone_data->total_clones >= clone_data->clone_max) {
	// If we've already used all available instances, this is an orphan
	as_orphan = TRUE;
	}

	// Allocate instance numbers in numerical order (starting at 0)
	inc_num = pcmk__itoa(clone_data->total_clones);
	inc_max = pcmk__itoa(clone_data->clone_max);

	child_copy = copy_xml(clone_data->xml_obj_child);

	crm_xml_add(child_copy, XML_RSC_ATTR_INCARNATION, inc_num);

	if (pe__unpack_resource(child_copy, &child_rsc, rsc,
	data_set) != pcmk_rc_ok) {
	goto bail;
	}
	/* child_rsc->globally_unique = rsc->globally_unique; */

	CRM_ASSERT(child_rsc);
	clone_data->total_clones += 1;
	pe_rsc_trace(child_rsc, "Setting clone attributes for: %s", child_rsc->id);
	rsc->children = g_list_append(rsc->children, child_rsc);
	if (as_orphan) {
	pe__set_resource_flags_recursive(child_rsc, pcmk_rsc_removed);
	}

	add_hash_param(child_rsc->meta, XML_RSC_ATTR_INCARNATION_MAX, inc_max);
	pe_rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);

	bail:
	free(inc_num);
	free(inc_max);

	return child_rsc;
	}

	gboolean
	clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	int lpc = 0;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = rsc->xml;
	clone_variant_data_t *clone_data = NULL;

	const char *max_clones = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_MAX);
	const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	clone_data = calloc(1, sizeof(clone_variant_data_t));
	rsc->variant_opaque = clone_data;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	const char *promoted_max = NULL;
	const char *promoted_node_max = NULL;

	promoted_max = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTED_MAX);
	if (promoted_max == NULL) {
	// @COMPAT deprecated since 2.0.0
	promoted_max = g_hash_table_lookup(rsc->meta,
	PCMK_XA_PROMOTED_MAX_LEGACY);
	}

	promoted_node_max = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTED_NODEMAX);
	if (promoted_node_max == NULL) {
	// @COMPAT deprecated since 2.0.0
	promoted_node_max =
	g_hash_table_lookup(rsc->meta,
	PCMK_XA_PROMOTED_NODE_MAX_LEGACY);
	}

	// Use 1 as default but 0 for minimum and invalid
	if (promoted_max == NULL) {
	clone_data->promoted_max = 1;
	} else {
	pcmk__scan_min_int(promoted_max, &(clone_data->promoted_max), 0);
	}

	// Use 1 as default but 0 for minimum and invalid
	if (promoted_node_max == NULL) {
	clone_data->promoted_node_max = 1;
	} else {
	pcmk__scan_min_int(promoted_node_max,
	&(clone_data->promoted_node_max), 0);
	}
	}

	// Implied by calloc()
	/* clone_data->xml_obj_child = NULL; */

	// Use 1 as default but 0 for minimum and invalid
	if (max_clones_node == NULL) {
	clone_data->clone_node_max = 1;
	} else {
	pcmk__scan_min_int(max_clones_node, &(clone_data->clone_node_max), 0);
	}

	/* Use number of nodes (but always at least 1, which is handy for crm_verify
	* for a CIB without nodes) as default, but 0 for minimum and invalid
	*/
	if (max_clones == NULL) {
	clone_data->clone_max = QB_MAX(1, g_list_length(data_set->nodes));
	} else {
	pcmk__scan_min_int(max_clones, &(clone_data->clone_max), 0);
	}

	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_ORDERED))) {
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", rsc->id,
	clone_data->flags,
	pe__clone_ordered,
	"pe__clone_ordered");
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_node_max > 1)) {

	pcmk__config_err("Ignoring " XML_RSC_ATTR_PROMOTED_MAX " for %s "
	"because anonymous clones support only one instance "
	"per node", rsc->id);
	clone_data->clone_node_max = 1;
	}

	pe_rsc_trace(rsc, "Options for %s", rsc->id);
	pe_rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
	pe_rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
	pe_rsc_trace(rsc, "\tClone is unique: %s",
	pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	pe_rsc_trace(rsc, "\tClone is promotable: %s",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable));

	// Clones may contain a single group or primitive
	for (a_child = pcmk__xe_first_child(xml_obj); a_child != NULL;
	a_child = pcmk__xe_next(a_child)) {

	if (pcmk__str_any_of((const char *)a_child->name, XML_CIB_TAG_RESOURCE, XML_CIB_TAG_GROUP, NULL)) {
	clone_data->xml_obj_child = a_child;
	break;
	}
	}

	if (clone_data->xml_obj_child == NULL) {
	pcmk__config_err("%s has nothing to clone", rsc->id);
	return FALSE;
	}

	/*
	* Make clones ever so slightly sticky by default
	*
	* This helps ensure clone instances are not shuffled around the cluster
	* for no benefit in situations when pre-allocation is not appropriate
	*/
	if (g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_STICKINESS) == NULL) {
	add_hash_param(rsc->meta, XML_RSC_ATTR_STICKINESS, "1");
	}

	/* This ensures that the globally-unique value always exists for children to
	* inherit when being unpacked, as well as in resource agents' environment.
	*/
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE,
	pe__rsc_bool_str(rsc, pcmk_rsc_unique));

	if (clone_data->clone_max <= 0) {
	/* Create one child instance so that unpack_find_resource() will hook up
	* any orphans up to the parent correctly.
	*/
	if (pe__create_clone_child(rsc, data_set) == NULL) {
	return FALSE;
	}

	} else {
	// Create a child instance for each available instance number
	for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
	if (pe__create_clone_child(rsc, data_set) == NULL) {
	return FALSE;
	}
	}
	}

	pe_rsc_trace(rsc, "Added %d children to resource %s...", clone_data->clone_max, rsc->id);
	return TRUE;
	}

	gboolean
	clone_active(pe_resource_t * rsc, gboolean all)
	{
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean child_active = child_rsc->fns->active(child_rsc, all);

	if (all == FALSE && child_active) {
	return TRUE;
	} else if (all && child_active == FALSE) {
	return FALSE;
	}
	}

	if (all) {
	return TRUE;
	} else {
	return FALSE;
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	short_print(const char list, const char prefix, const char *type,
	const char suffix, long options, void print_data)
	{
	if(suffix == NULL) {
	suffix = "";
	}

	if (!pcmk__str_empty(list)) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	status_print("%s%s: [ %s ]%s", prefix, type, list, suffix);

	if (options & pe_print_html) {
	status_print("</li>\n");

	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}

	}
	}

	static const char *
	configured_role_str(pe_resource_t * rsc)
	{
	const char *target_role = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_TARGET_ROLE);

	if ((target_role == NULL) && rsc->children && rsc->children->data) {
	target_role = g_hash_table_lookup(((pe_resource_t*)rsc->children->data)->meta,
	XML_RSC_ATTR_TARGET_ROLE);
	}
	return target_role;
	}

	static enum rsc_role_e
	configured_role(pe_resource_t * rsc)
	{
	const char *target_role = configured_role_str(rsc);

	if (target_role) {
	return text2role(target_role);
	}
	return pcmk_role_unknown;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	clone_print_xml(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	char *child_text = crm_strdup_printf("%s ", pre_text);
	const char *target_role = configured_role_str(rsc);
	GList *gIter = rsc->children;

	status_print("%s<clone ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
	status_print("multi_state=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable));
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</clone>\n", pre_text);
	free(child_text);
	}

	bool
	is_set_recursive(const pe_resource_t *rsc, long long flag, bool any)
	{
	GList *gIter;
	bool all = !any;

	if (pcmk_is_set(rsc->flags, flag)) {
	if(any) {
	return TRUE;
	}
	} else if(all) {
	return FALSE;
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	if(is_set_recursive(gIter->data, flag, any)) {
	if(any) {
	return TRUE;
	}

	} else if(all) {
	return FALSE;
	}
	}

	if(all) {
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	clone_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	GString *list_text = NULL;
	char *child_text = NULL;
	GString *stopped_list = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	clone_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_clone_variant_data(clone_data, rsc);

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sClone Set: %s [%s]%s%s%s",
	pre_text ? pre_text : "", rsc->id, ID(clone_data->xml_obj_child),
	pcmk_is_set(rsc->flags, pcmk_rsc_promotable)? " (promotable)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (options & pe_print_clone_details) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(options, pe_print_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(options, pe_print_clone_active)) {

	pcmk__add_word(&stopped_list, 1024, child_rsc->id);
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);

	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	short_print((const char *) list_text->str, child_text,
	PROMOTED_INSTANCES, NULL, options, print_data);
	g_string_truncate(list_text, 0);
	}
	g_list_free(promoted_list);

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES " (target-role)", NULL,
	options, print_data);
	} else {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES, NULL, options, print_data);
	}

	} else {
	short_print((const char *) list_text->str, child_text, "Started",
	NULL, options, print_data);
	}
	}

	g_list_free(started_list);

	if (!pcmk_is_set(options, pe_print_clone_active)) {
	const char *state = "Stopped";
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped list for non-unique clones */
	if (stopped_list != NULL) {
	g_string_truncate(stopped_list, 0);
	}

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pe_node_t node = (pe_node_t )nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL) {
	pcmk__add_word(&stopped_list, 1024, node->details->uname);
	}
	}
	g_list_free(list);
	}

	if (stopped_list != NULL) {
	short_print((const char *) stopped_list->str, child_text, state,
	NULL, options, print_data);
	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (stopped_list != NULL) {
	g_string_free(stopped_list, TRUE);
	}
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__clone_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );


	const char *desc = NULL;
	GList *gIter = rsc->children;
	GList *all = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;



	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	all = g_list_prepend(all, (gpointer) "*");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);
	rc = pe__name_and_nvpairs_xml(out, true, "clone", 10,
	"id", rsc->id,
	"multi_state",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable),
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	- "maintenance", pe__rsc_bool_str(rsc, pe_rsc_maintenance),
	+ "maintenance",
	+ pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"disabled", pcmk__btoa(pe__resource_is_disabled(rsc)),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"target_role", configured_role_str(rsc),
	"description", desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out);
	}

	g_list_free(all);
	return rc;
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__clone_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GHashTable *stopped = NULL;

	GString *list_text = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	const char *desc = NULL;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_clone_variant_data(clone_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(show_opts, pcmk_show_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(show_opts, pcmk_show_clone_detail)
	&& pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	g_hash_table_insert(stopped, strdup(child_rsc->id), strdup("Stopped"));
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);

	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	GList *all = NULL;

	clone_header(out, &rc, rsc, clone_data, desc);

	/* Print every resource that's a child of this clone. */
	all = g_list_prepend(all, (gpointer) "*");
	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	g_list_free(all);
	}
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return pcmk_rc_ok;
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(promoted_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	out->list_item(out, NULL, PROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	g_string_truncate(list_text, 0);
	}

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(started_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	out->list_item(out, NULL,
	UNPROMOTED_INSTANCES " (target-role): [ %s ]",
	(const char *) list_text->str);
	} else {
	out->list_item(out, NULL, UNPROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	}

	} else {
	out->list_item(out, NULL, "Started: [ %s ]",
	(const char *) list_text->str);
	}
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped table for non-unique clones */
	if (stopped != NULL) {
	g_hash_table_destroy(stopped);
	stopped = NULL;
	}

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pe_node_t node = (pe_node_t )nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
	pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
	const char *state = "Stopped";

	if (configured_role(rsc) == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	g_hash_table_insert(stopped, strdup(node->details->uname),
	crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
	} else {
	g_hash_table_insert(stopped, strdup(node->details->uname),
	strdup(state));
	}
	}
	}
	g_list_free(list);
	}

	if (stopped != NULL) {
	GList *list = sorted_hash_table_values(stopped);

	clone_header(out, &rc, rsc, clone_data, desc);

	for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
	const char *status = status_iter->data;
	GList *nodes = nodes_with_status(stopped, status);
	GString *nodes_str = node_list_to_str(nodes);

	if (nodes_str != NULL) {
	if (nodes_str->len > 0) {
	out->list_item(out, NULL, "%s: [ %s ]", status,
	(const char *) nodes_str->str);
	}
	g_string_free(nodes_str, TRUE);
	}

	g_list_free(nodes);
	}

	g_list_free(list);
	g_hash_table_destroy(stopped);

	/* If there are no instances of this clone (perhaps because there are no
	* nodes configured), simply output the clone header by itself. This can
	* come up in PCS testing.
	*/
	} else if (active_instances == 0) {
	clone_header(out, &rc, rsc, clone_data, desc);
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	void
	clone_free(pe_resource_t * rsc)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pe_rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	free_xml(child_rsc->xml);
	child_rsc->xml = NULL;
	/* There could be a saved unexpanded xml */
	free_xml(child_rsc->orig_xml);
	child_rsc->orig_xml = NULL;
	child_rsc->fns->free(child_rsc);
	}

	g_list_free(rsc->children);

	if (clone_data) {
	CRM_ASSERT(clone_data->demote_notify == NULL);
	CRM_ASSERT(clone_data->stop_notify == NULL);
	CRM_ASSERT(clone_data->start_notify == NULL);
	CRM_ASSERT(clone_data->promote_notify == NULL);
	}

	common_free(rsc);
	}

	enum rsc_role_e
	clone_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e clone_role = pcmk_role_unknown;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, current);

	if (a_role > clone_role) {
	clone_role = a_role;
	}
	}

	pe_rsc_trace(rsc, "%s role: %s", rsc->id, role2text(clone_role));
	return clone_role;
	}

	/*!
	* \internal
	* \brief Check whether a clone has an instance for every node
	*
	* \param[in] rsc Clone to check
	* \param[in] data_set Cluster state
	*/
	bool
	pe__is_universal_clone(const pe_resource_t *rsc,
	const pe_working_set_t *data_set)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	if (clone_data->clone_max == g_list_length(data_set->nodes)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gboolean
	pe__clone_is_filtered(const pe_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	clone_variant_data_t *clone_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_clone_variant_data(clone_data, rsc);
	passes = pcmk__str_in_list(ID(clone_data->xml_obj_child), only_rsc, pcmk__str_star_matches);

	if (!passes) {
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pe_resource_t *child_rsc = NULL;

	child_rsc = (const pe_resource_t *) iter->data;
	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}
	}
	return !passes;
	}

	const char *
	pe__clone_child_id(const pe_resource_t *rsc)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);
	return ID(clone_data->xml_obj_child);
	}

	/*!
	* \internal
	* \brief Check whether a clone is ordered
	*
	* \param[in] clone Clone resource to check
	*
	* \return true if clone is ordered, otherwise false
	*/
	bool
	pe__clone_is_ordered(const pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	return pcmk_is_set(clone_data->flags, pe__clone_ordered);
	}

	/*!
	* \internal
	* \brief Set a clone flag
	*
	* \param[in,out] clone Clone resource to set flag for
	* \param[in] flag Clone flag to set
	*
	* \return Standard Pacemaker return code (either pcmk_rc_ok if flag was not
	* already set or pcmk_rc_already if it was)
	*/
	int
	pe__set_clone_flag(pe_resource_t *clone, enum pe__clone_flags flag)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	if (pcmk_is_set(clone_data->flags, flag)) {
	return pcmk_rc_already;
	}
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", clone->id,
	clone_data->flags, flag, "flag");
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a clone flag is set
	*
	* \param[in] group Clone resource to check
	* \param[in] flags Flag or flags to check
	*
	* \return \c true if all \p flags are set for \p clone, otherwise \c false
	*/
	bool
	pe__clone_flag_is_set(const pe_resource_t *clone, uint32_t flags)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	CRM_ASSERT(clone_data != NULL);

	return pcmk_all_flags_set(clone_data->flags, flags);
	}

	/*!
	* \internal
	* \brief Create pseudo-actions needed for promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[in] any_promoting Whether any instances will be promoted
	* \param[in] any_demoting Whether any instance will be demoted
	*/
	void
	pe__create_promotable_pseudo_ops(pe_resource_t *clone, bool any_promoting,
	bool any_demoting)
	{
	pe_action_t *action = NULL;
	pe_action_t *action_complete = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	// Create a "promote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTE,
	!any_promoting, true);

	// Create a "promoted" action for when all promotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTED,
	!any_promoting, true);
	action_complete->priority = INFINITY;

	// Create notification pseudo-actions for promotion
	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_PROMOTE,
	action,
	action_complete);
	}

	// Create a "demote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTE,
	!any_demoting, true);

	// Create a "demoted" action for when all demotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTED,
	!any_demoting, true);
	action_complete->priority = INFINITY;

	// Create notification pseudo-actions for demotion
	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_DEMOTE,
	action,
	action_complete);

	if (clone_data->promote_notify != NULL) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->start_notify->post_done,
	clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->start_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->stop_notify->pre,
	pe_order_optional);
	}
	}
	}

	/*!
	* \internal
	* \brief Create all notification data and actions for a clone
	*
	* \param[in,out] clone Clone to create notifications for
	*/
	void
	pe__create_clone_notifications(pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__create_action_notifications(clone, clone_data->start_notify);
	pe__create_action_notifications(clone, clone_data->stop_notify);
	pe__create_action_notifications(clone, clone_data->promote_notify);
	pe__create_action_notifications(clone, clone_data->demote_notify);
	}

	/*!
	* \internal
	* \brief Free all notification data for a clone
	*
	* \param[in,out] clone Clone to free notification data for
	*/
	void
	pe__free_clone_notification_data(pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__free_action_notification_data(clone_data->demote_notify);
	clone_data->demote_notify = NULL;

	pe__free_action_notification_data(clone_data->stop_notify);
	clone_data->stop_notify = NULL;

	pe__free_action_notification_data(clone_data->start_notify);
	clone_data->start_notify = NULL;

	pe__free_action_notification_data(clone_data->promote_notify);
	clone_data->promote_notify = NULL;
	}

	/*!
	* \internal
	* \brief Create pseudo-actions for clone start/stop notifications
	*
	* \param[in,out] clone Clone to create pseudo-actions for
	* \param[in,out] start Start action for \p clone
	* \param[in,out] stop Stop action for \p clone
	* \param[in,out] started Started action for \p clone
	* \param[in,out] stopped Stopped action for \p clone
	*/
	void
	pe__create_clone_notif_pseudo_ops(pe_resource_t *clone,
	pe_action_t start, pe_action_t started,
	pe_action_t stop, pe_action_t stopped)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	if (clone_data->start_notify == NULL) {
	clone_data->start_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_START,
	start, started);
	}

	if (clone_data->stop_notify == NULL) {
	clone_data->stop_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_STOP,
	stop, stopped);
	if ((clone_data->start_notify != NULL)
	&& (clone_data->stop_notify != NULL)) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->start_notify->pre, pe_order_optional);
	}
	}
	}

	/*!
	* \internal
	* \brief Get maximum clone resource instances per node
	*
	* \param[in] rsc Clone resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__clone_max_per_node(const pe_resource_t *rsc)
	{
	const char *max_clones_node = NULL;
	int max_instances = 1;

	CRM_ASSERT(pe_rsc_is_clone(rsc));
	max_clones_node = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INCARNATION_NODEMAX);
	if (max_clones_node != NULL) {
	pcmk__scan_min_int(max_clones_node, &max_instances, 0);
	}
	return (unsigned int) max_instances;
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index c2a54a389b..406fc904dc 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1185 +1,1185 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pe_node_t active_node(const pe_resource_t rsc, unsigned int *count_all,
	unsigned int *count_clean);

	resource_object_functions_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pcmk_rsc_variant_primitive;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pcmk_rsc_variant_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pcmk_rsc_variant_bundle;
	}

	return pcmk_rsc_variant_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pe_resource_t * rsc, pe_rule_eval_data_t * rule_data, GHashTable * meta_hash, pe_working_set_t * data_set)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pe_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS,
	rule_data, parent_orig_meta, NULL, FALSE, data_set);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
	pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, data_set);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pe_resource_t rsc,
	const pe_node_t node, pe_working_set_t data_set)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, data_set);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *clone = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pe_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	XML_ATTR_ID, template_ref);
	if (template == NULL) {
	pe_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, XML_ATTR_ID, id);

	clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
	if(clone) {
	crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
	}

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	/* Disable multi-level templates for now */
	/*if(unpack_template(new_xml, expanded_xml, data_set) == FALSE) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;

	return FALSE;
	} */

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode * xml_obj, pe_working_set_t * data_set)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for processing resource list of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	if (add_tag_ref(data_set->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pe_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
	/* @TODO in some future version, pe_warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] data_set Cluster working set
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p data_set is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pe_resource_t rsc, const pe_node_t node,
	pe_working_set_t *data_set)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (data_set == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, data_set);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pe_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pe__set_resource_flags(rsc,
	pe_rsc_needs_fencing\|pe_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pe_rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pe_resource_t *rsc)
	{
	const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pe_warn_once(pe_wo_upstart,
	"Support for Upstart resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pe_warn_once(pe_wo_nagios,
	"Support for Nagios resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pe_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pe_resource_t *rsc,
	pe_resource_t parent, pe_working_set_t data_set)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (data_set != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = data_set->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pe_err("Ignoring <%s> configuration without " XML_ATTR_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, data_set) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pe_resource_t));
	if (*rsc == NULL) {
	crm_crit("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = data_set;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, data_set->input);

	(rsc)->variant = get_resource_type((const char ) (*rsc)->xml->name);
	if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
	pe_err("Ignoring resource '%s' of unknown type '%s'",
	id, (*rsc)->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, data_set);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, data_set); // \deprecated

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pcmk_rsc_runnable\|pcmk_rsc_unassigned);

	if (!pcmk_is_set(data_set->flags, pcmk_sched_in_maintenance)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = pcmk_role_stopped;
	(*rsc)->next_role = pcmk_role_unknown;

	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* allow-migrate=false.
	*/
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	- pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	+ pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}
	if (pcmk_is_set(data_set->flags, pcmk_sched_in_maintenance)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	- pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	+ pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pe_rsc_trace((*rsc), "%s dependency restart handling: restart",
	(*rsc)->id);
	pe_warn_once(pe_wo_restart_type,
	"Support for restart-type is deprecated and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pe_rsc_trace((*rsc), "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_stop;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_block;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pe_warn("%s is not a valid value for " XML_RSC_ATTR_MULTIPLE
	", using default of \"stop_start\"", value);
	}
	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: "
	"stop/start", (*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pe_warn_once(pe_wo_neg_threshold,
	PCMK_META_MIGRATION_THRESHOLD
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe__set_working_set_flags(data_set, pcmk_sched_have_fencing);
	pe__set_resource_flags(*rsc, pcmk_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	// Stored as seconds
	(*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, data_set);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* reconnect_interval.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	(*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
	/* we want to override any default failure_timeout in use when remote
	* reconnect_interval is in use. */
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pe_rsc_trace((rsc), "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != pcmk_role_unknown? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, data_set) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(data_set->flags, pcmk_sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", data_set);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default", data_set);
	}

	pe_rsc_trace((rsc), "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, data_set);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, data_set) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pe_resource_t child, pe_resource_t rsc)
	{
	pe_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pe_resource_t *
	uber_parent(pe_resource_t * rsc)
	{
	pe_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->parent != NULL)
	&& (parent->parent->variant != pcmk_rsc_variant_bundle)) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pe_resource_t *
	pe__const_top_resource(const pe_resource_t *rsc, bool include_bundle)
	{
	const pe_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle
	&& (parent->parent->variant == pcmk_rsc_variant_bundle)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pe_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pe_rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {

	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pe_rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pe_resource_t rsc, pe_node_t node,
	pe_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of resource_object_functions_t:active_node()
	static pe_node_t *
	active_node(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pe_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pe_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pe_node_t *
	pe__find_active_requires(const pe_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pe_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pe_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role > pcmk_role_stopped)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pe_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pe_rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role), why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/group.c b/lib/pengine/group.c
	index 7382c399c3..9a140bcb7c 100644
	--- a/lib/pengine/group.c
	+++ b/lib/pengine/group.c
	@@ -1,536 +1,536 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	typedef struct group_variant_data_s {
	pe_resource_t *last_child; // Last group member
	uint32_t flags; // Group of enum pe__group_flags
	} group_variant_data_t;

	/*!
	* \internal
	* \brief Get a group's last member
	*
	* \param[in] group Group resource to check
	*
	* \return Last member of \p group if any, otherwise NULL
	*/
	pe_resource_t *
	pe__last_group_member(const pe_resource_t *group)
	{
	if (group != NULL) {
	CRM_CHECK((group->variant == pcmk_rsc_variant_group)
	&& (group->variant_opaque != NULL), return NULL);
	return ((group_variant_data_t *) group->variant_opaque)->last_child;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a group flag is set
	*
	* \param[in] group Group resource to check
	* \param[in] flags Flag or flags to check
	*
	* \return true if all \p flags are set for \p group, otherwise false
	*/
	bool
	pe__group_flag_is_set(const pe_resource_t *group, uint32_t flags)
	{
	group_variant_data_t *group_data = NULL;

	CRM_CHECK((group != NULL) && (group->variant == pcmk_rsc_variant_group)
	&& (group->variant_opaque != NULL), return false);
	group_data = (group_variant_data_t *) group->variant_opaque;
	return pcmk_all_flags_set(group_data->flags, flags);
	}

	/*!
	* \internal
	* \brief Set a (deprecated) group flag
	*
	* \param[in,out] group Group resource to check
	* \param[in] option Name of boolean configuration option
	* \param[in] flag Flag to set if \p option is true (which is default)
	* \param[in] wo_bit "Warn once" flag to use for deprecation warning
	*/
	static void
	set_group_flag(pe_resource_t group, const char option, uint32_t flag,
	uint32_t wo_bit)
	{
	const char *value_s = NULL;
	int value = 0;

	value_s = g_hash_table_lookup(group->meta, option);

	// We don't actually need the null check but it speeds up the common case
	if ((value_s == NULL) \|\| (crm_str_to_boolean(value_s, &value) < 0)
	\|\| (value != 0)) {

	((group_variant_data_t *) group->variant_opaque)->flags \|= flag;

	} else {
	pe_warn_once(wo_bit,
	"Support for the '%s' group meta-attribute is deprecated "
	"and will be removed in a future release "
	"(use a resource set instead)", option);
	}
	}

	static int
	inactive_resources(pe_resource_t *rsc)
	{
	int retval = 0;

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (!child_rsc->fns->active(child_rsc, TRUE)) {
	retval++;
	}
	}

	return retval;
	}

	static void
	group_header(pcmk__output_t out, int rc, const pe_resource_t *rsc,
	int n_inactive, bool show_inactive, const char *desc)
	{
	GString *attrs = NULL;

	if (n_inactive > 0 && !show_inactive) {
	attrs = g_string_sized_new(64);
	g_string_append_printf(attrs, "%d member%s inactive", n_inactive,
	pcmk__plural_s(n_inactive));
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
	}

	- if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	+ if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
	}

	if (attrs != NULL) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Resource Group: %s (%s)%s%s%s",
	rsc->id,
	(const char *) attrs->str, desc ? " (" : "",
	desc ? desc : "", desc ? ")" : "");
	g_string_free(attrs, TRUE);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Resource Group: %s%s%s%s",
	rsc->id,
	desc ? " (" : "", desc ? desc : "",
	desc ? ")" : "");
	}
	}

	static bool
	skip_child_rsc(pe_resource_t rsc, pe_resource_t child, gboolean parent_passes,
	GList *only_rsc, uint32_t show_opts)
	{
	bool star_list = pcmk__list_of_1(only_rsc) &&
	pcmk__str_eq("*", g_list_first(only_rsc)->data, pcmk__str_none);
	bool child_filtered = child->fns->is_filtered(child, only_rsc, FALSE);
	bool child_active = child->fns->active(child, FALSE);
	bool show_inactive = pcmk_is_set(show_opts, pcmk_show_inactive_rscs);

	/* If the resource is in only_rsc by name (so, ignoring "*") then allow
	* it regardless of if it's active or not.
	*/
	if (!star_list && !child_filtered) {
	return false;

	} else if (!child_filtered && (child_active \|\| show_inactive)) {
	return false;

	} else if (parent_passes && (child_active \|\| show_inactive)) {
	return false;

	}

	return true;
	}

	gboolean
	group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = rsc->xml;
	xmlNode *xml_native_rsc = NULL;
	group_variant_data_t *group_data = NULL;
	const char *clone_id = NULL;

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	group_data = calloc(1, sizeof(group_variant_data_t));
	group_data->last_child = NULL;
	rsc->variant_opaque = group_data;

	// @COMPAT These are deprecated since 2.1.5
	set_group_flag(rsc, XML_RSC_ATTR_ORDERED, pe__group_ordered,
	pe_wo_group_order);
	set_group_flag(rsc, "collocated", pe__group_colocated, pe_wo_group_coloc);

	clone_id = crm_element_value(rsc->xml, XML_RSC_ATTR_INCARNATION);

	for (xml_native_rsc = pcmk__xe_first_child(xml_obj); xml_native_rsc != NULL;
	xml_native_rsc = pcmk__xe_next(xml_native_rsc)) {

	if (pcmk__str_eq((const char *)xml_native_rsc->name,
	XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	pe_resource_t *new_rsc = NULL;

	crm_xml_add(xml_native_rsc, XML_RSC_ATTR_INCARNATION, clone_id);
	if (pe__unpack_resource(xml_native_rsc, &new_rsc, rsc,
	data_set) != pcmk_rc_ok) {
	continue;
	}

	rsc->children = g_list_append(rsc->children, new_rsc);
	group_data->last_child = new_rsc;
	pe_rsc_trace(rsc, "Added %s member %s", rsc->id, new_rsc->id);
	}
	}

	if (rsc->children == NULL) {
	/* The schema does not allow empty groups, but if validation is
	* disabled, we allow them (members can be added later).
	*
	* @COMPAT At a major release bump, we should consider this a failure so
	* that group methods can assume children is not NULL, and there
	* are no strange effects from phantom groups due to their
	* presence or meta-attributes.
	*/
	pcmk__config_warn("Group %s will be ignored because it does not have "
	"any members", rsc->id);
	}
	return TRUE;
	}

	gboolean
	group_active(pe_resource_t * rsc, gboolean all)
	{
	gboolean c_all = TRUE;
	gboolean c_any = FALSE;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (child_rsc->fns->active(child_rsc, all)) {
	c_any = TRUE;
	} else {
	c_all = FALSE;
	}
	}

	if (c_any == FALSE) {
	return FALSE;
	} else if (all && c_all == FALSE) {
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	group_print_xml(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	GList *gIter = rsc->children;
	char *child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%s<group " XML_ATTR_ID "=\"%s\" ", pre_text, rsc->id);
	status_print("number_resources=\"%d\" ", g_list_length(rsc->children));
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</group>\n", pre_text);
	free(child_text);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	group_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	char *child_text = NULL;
	GList *gIter = rsc->children;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	group_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sResource Group: %s", pre_text ? pre_text : "", rsc->id);

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	if (options & pe_print_brief) {
	print_rscs_brief(rsc->children, child_text, options, print_data, TRUE);

	} else {
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("group", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__group_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	GList *gIter = rsc->children;

	int rc = pcmk_rc_no_output;

	gboolean parent_passes = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (skip_child_rsc(rsc, child_rsc, parent_passes, only_rsc, show_opts)) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	char *count = pcmk__itoa(g_list_length(gIter));
	- const char *maint_s = pe__rsc_bool_str(rsc, pe_rsc_maintenance);
	+ const char *maint_s = pe__rsc_bool_str(rsc, pcmk_rsc_maintenance);
	const char *managed_s = pe__rsc_bool_str(rsc, pcmk_rsc_managed);
	const char *disabled_s = pcmk__btoa(pe__resource_is_disabled(rsc));

	rc = pe__name_and_nvpairs_xml(out, true, "group", 5,
	XML_ATTR_ID, rsc->id,
	"number_resources", count,
	"maintenance", maint_s,
	"managed", managed_s,
	"disabled", disabled_s,
	"description", desc);
	free(count);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	out->message(out, crm_map_element_name(child_rsc->xml), show_opts, child_rsc,
	only_node, only_rsc);
	}

	if (rc == pcmk_rc_ok) {
	pcmk__output_xml_pop_parent(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("group", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__group_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	int rc = pcmk_rc_no_output;

	gboolean parent_passes = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	gboolean active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(rsc->children, only_rsc);

	if (rscs != NULL) {
	group_header(out, &rc, rsc, !active && partially_active ? inactive_resources(rsc) : 0,
	pcmk_is_set(show_opts, pcmk_show_inactive_rscs), desc);
	pe__rscs_brief_output(out, rscs, show_opts \| pcmk_show_inactive_rscs);

	rc = pcmk_rc_ok;
	g_list_free(rscs);
	}

	} else {
	for (GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (skip_child_rsc(rsc, child_rsc, parent_passes, only_rsc, show_opts)) {
	continue;
	}

	group_header(out, &rc, rsc, !active && partially_active ? inactive_resources(rsc) : 0,
	pcmk_is_set(show_opts, pcmk_show_inactive_rscs), desc);
	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, only_rsc);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	return rc;
	}

	void
	group_free(pe_resource_t * rsc)
	{
	CRM_CHECK(rsc != NULL, return);

	pe_rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pe_rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	child_rsc->fns->free(child_rsc);
	}

	pe_rsc_trace(rsc, "Freeing child list");
	g_list_free(rsc->children);

	common_free(rsc);
	}

	enum rsc_role_e
	group_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e group_role = pcmk_role_unknown;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	enum rsc_role_e role = child_rsc->fns->state(child_rsc, current);

	if (role > group_role) {
	group_role = role;
	}
	}

	pe_rsc_trace(rsc, "%s role: %s", rsc->id, role2text(group_role));
	return group_role;
	}

	gboolean
	pe__group_is_filtered(const pe_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;

	if (check_parent
	&& pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc,
	false)),
	only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else if (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pe_resource_t child_rsc = (const pe_resource_t ) iter->data;

	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get maximum group resource instances per node
	*
	* \param[in] rsc Group resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__group_max_per_node(const pe_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));
	return 1U;
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 895bb71a5f..47a4bfb748 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1446 +1,1447 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pe_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pe_resource_t * rsc, pe_node_t * node, gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == pcmk_role_promoted) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pe_rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pe__node_name(node), node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pe_rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s) "
	"from guest node %s",
	pe__node_name(a_node), a_node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "",
	pe__node_name(node));
	}
	}
	}

	void
	native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pe_rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pcmk_rsc_variant_primitive) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& node->details->maintenance) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	- pe__set_resource_flags(rsc, pe_rsc_maintenance);
	+ pe__set_resource_flags(rsc, pcmk_rsc_maintenance);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_resource_t *p = rsc->parent;

	pe_rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default", data_set);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	case pcmk_multiply_active_stop:
	{
	GHashTableIter gIter;
	pe_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case pcmk_multiply_active_block:
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);

	/* If the resource belongs to a group or bundle configured with
	* multiple-active=block, block the entire entity.
	*/
	if (rsc->parent
	&& ((rsc->parent->variant == pcmk_rsc_variant_group)
	\|\| (rsc->parent->variant == pcmk_rsc_variant_bundle))
	&& (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__clear_resource_flags(child, pcmk_rsc_managed);
	pe__set_resource_flags(child, pcmk_rsc_blocked);
	}
	}
	break;

	// pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
	default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pe__node_name(node),
	recovery2text(rsc->recovery_type));

	} else {
	pe_rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pe__node_name(node));
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, data_set, FALSE);
	}
	}

	static void
	recursive_clear_unique(pe_resource_t *rsc, gpointer user_data)
	{
	pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_resource_t *parent = uber_parent(rsc);
	const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, data_set);

	/* Clear globally-unique on the parent and all its descendants unpacked
	* so far (clearing the parent should make any future children unpacking
	* correct). We have to clear this resource explicitly because it isn't
	* hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pcmk_rsc_promotable)) {

	pe_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pe_resource_t rsc, const pe_node_t node, int flags)
	{
	pe_rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pe__node_name(node));

	if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->running_on != NULL)) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pe_node_t loc = (pe_node_t ) iter->data;

	if (loc->details == node->details) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
	&& (rsc->running_on == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->allocated_to != NULL)
	&& (rsc->allocated_to->details == node->details)) {
	return true;
	}
	return false;
	}

	pe_resource_t *
	native_find_rsc(pe_resource_t * rsc, const char id, const pe_node_t on_node,
	int flags)
	{
	bool match = false;
	pe_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_history)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
	\|\| (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, data_set);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	pcmk__str_update(&value_copy, value);
	return value_copy;
	}

	gboolean
	native_active(pe_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pe_rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	} else if (!a_node->details->online
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pe__node_name(a_node));
	} else {
	pe_rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pe_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_PROMOTE,
	pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_DEMOTE,
	pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(const pe_resource_t *rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(const pe_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == pcmk_role_started)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {

	role = pcmk_role_unpromoted;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pe_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	native_print_xml(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_removed));
	status_print("blocked=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_blocked));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" " XML_ATTR_ID "=\"%s\" "
	"cached=\"%s\"/>\n",
	pre_text, pcmk__s(node->details->uname, ""),
	node->details->id, pcmk__btoa(!node->details->online));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pe_resource_t rsc, const char name,
	const pe_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > pcmk_role_unpromoted) {
	pcmk__add_word(&outstr, 0, role2text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pe__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == pcmk_role_stopped) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent the
	* resource from becoming promoted).
	*/
	if (target_role_e == pcmk_role_stopped) {
	have_flags = add_output_flag(outstr, "disabled", have_flags);

	} else if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)
	&& (target_role_e == pcmk_role_unpromoted)) {
	have_flags = add_output_flag(outstr, "target-role:", have_flags);
	g_string_append(outstr, target_role);
	}
	}

	// Blocked or maintenance implies unmanaged
	- if (pcmk_any_flags_set(rsc->flags, pcmk_rsc_blocked\|pe_rsc_maintenance)) {
	+ if (pcmk_any_flags_set(rsc->flags,
	+ pcmk_rsc_blocked\|pcmk_rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	- } else if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	+ } else if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t n = (pe_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pe_resource_t rsc,
	const char name, const pe_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	cl = "rsc-failed";

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pe_resource_t rsc,
	const char name, const pe_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	common_print(pe_resource_t rsc, const char pre_text, const char *name,
	const pe_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t n = (pe_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", pe__node_name(n));

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", pe__node_name(n));

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, pe__node_name(n));

	} else {
	status_print("%s", pe__node_name(n));
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	native_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const pe_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	const char *desc = NULL;
	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	const char *lock_node_name = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));

	if (rsc->lock_node != NULL) {
	lock_node_name = rsc->lock_node->details->uname;
	}

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
	"id", rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pcmk_rsc_removed),
	"blocked", pe__rsc_bool_str(rsc, pcmk_rsc_blocked),
	- "maintenance", pe__rsc_bool_str(rsc, pe_rsc_maintenance),
	+ "maintenance", pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL),
	"locked_to", lock_node_name,
	"description", desc);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	"name", node->details->uname,
	"id", node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pe_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pe_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pe_resource_t * rsc)
	{
	pe_rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pe_node_t *
	native_location(const pe_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pe__rsc_node argument instead of int current
	pe_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_node_t node = (pe_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pe_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	const pe_resource_t *up = pe__const_top_resource(rsc, true);

	return up->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pe_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
	return 1U;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Oct 29, 8:57 PM (1 d, 12 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 942639
Default Alt Text: (451 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions