No OneTemporary
Actions

Size

239 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/common/logging_internal.h b/include/crm/common/logging_internal.h
	index 7bbde5d62d..451770e790 100644
	--- a/include/crm/common/logging_internal.h
	+++ b/include/crm/common/logging_internal.h
	@@ -1,241 +1,239 @@
	/*
	* Copyright 2015-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_LOGGING_INTERNAL__H
	#define PCMK__CRM_COMMON_LOGGING_INTERNAL__H

	#include <glib.h>

	#include <crm/common/logging.h>
	#include <crm/common/output_internal.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/* Some warnings are too noisy when logged every time a given function is called
	* (for example, using a deprecated feature). As an alternative, we allow
	* warnings to be logged once per invocation of the calling program. Each of
	* those warnings needs a flag defined here.
	*/
	enum pcmk__warnings {
	pcmk__wo_blind = (1 << 0),
	- pcmk__wo_restart_type = (1 << 1),
	- pcmk__wo_role_after = (1 << 2),
	pcmk__wo_require_all = (1 << 4),
	pcmk__wo_order_score = (1 << 5),
	pcmk__wo_group_order = (1 << 11),
	pcmk__wo_group_coloc = (1 << 12),
	pcmk__wo_set_ordering = (1 << 15),
	pcmk__wo_rdisc_enabled = (1 << 16),
	pcmk__wo_op_attr_expr = (1 << 19),
	pcmk__wo_instance_defaults = (1 << 20),
	pcmk__wo_multiple_rules = (1 << 21),
	pcmk__wo_clone_master_max = (1 << 23),
	pcmk__wo_clone_master_node_max = (1 << 24),
	pcmk__wo_master_role = (1 << 26),
	pcmk__wo_slave_role = (1 << 27),
	};

	/*!
	* \internal
	* \brief Log a warning once per invocation of calling program
	*
	* \param[in] wo_flag enum pcmk__warnings value for this warning
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__warn_once(wo_flag, fmt...) do { \
	if (!pcmk_is_set(pcmk__warnings, wo_flag)) { \
	if (wo_flag == pcmk__wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pcmk__config_warn(fmt); \
	} \
	pcmk__warnings = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Warn-once", "logging", \
	pcmk__warnings, \
	(wo_flag), #wo_flag); \
	} \
	} while (0)

	typedef void (pcmk__config_error_func) (void ctx, const char *msg, ...)
	G_GNUC_PRINTF(2, 3);
	typedef void (pcmk__config_warning_func) (void ctx, const char *msg, ...)
	G_GNUC_PRINTF(2, 3);

	extern pcmk__config_error_func pcmk__config_error_handler;
	extern pcmk__config_warning_func pcmk__config_warning_handler;

	extern void *pcmk__config_error_context;
	extern void *pcmk__config_warning_context;

	void pcmk__set_config_error_handler(pcmk__config_error_func error_handler, void *error_context);
	void pcmk__set_config_warning_handler(pcmk__config_warning_func warning_handler, void *warning_context);

	/* Pacemaker library functions set this when a configuration error is found,
	* which turns on extra messages at the end of processing.
	*/
	extern bool pcmk__config_has_error;

	/* Pacemaker library functions set this when a configuration warning is found,
	* which turns on extra messages at the end of processing.
	*/
	extern bool pcmk__config_has_warning;

	/*!
	* \internal
	* \brief Log an error and make crm_verify return failure status
	*
	* \param[in] fmt... printf(3)-style format string and arguments
	*/
	#define pcmk__config_err(fmt...) do { \
	pcmk__config_has_error = true; \
	if (pcmk__config_error_handler == NULL) { \
	crm_err(fmt); \
	} else { \
	pcmk__config_error_handler(pcmk__config_error_context, fmt); \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Log a warning and make crm_verify return failure status
	*
	* \param[in] fmt... printf(3)-style format string and arguments
	*/
	#define pcmk__config_warn(fmt...) do { \
	pcmk__config_has_warning = true; \
	if (pcmk__config_warning_handler == NULL) { \
	crm_warn(fmt); \
	} else { \
	pcmk__config_warning_handler(pcmk__config_warning_context, fmt);\
	} \
	} while (0)

	/*!
	* \internal
	* \brief Execute code depending on whether trace logging is enabled
	*
	* This is similar to \p do_crm_log_unlikely() except instead of logging, it
	* selects one of two code blocks to execute.
	*
	* \param[in] if_action Code block to execute if trace logging is enabled
	* \param[in] else_action Code block to execute if trace logging is not enabled
	*
	* \note Neither \p if_action nor \p else_action can contain a \p break or
	* \p continue statement.
	*/
	#define pcmk__if_tracing(if_action, else_action) do { \
	static struct qb_log_callsite *trace_cs = NULL; \
	\
	if (trace_cs == NULL) { \
	trace_cs = qb_log_callsite_get(__func__, __FILE__, \
	"if_tracing", LOG_TRACE, \
	__LINE__, crm_trace_nonlog); \
	} \
	if (crm_is_callsite_active(trace_cs, LOG_TRACE, \
	crm_trace_nonlog)) { \
	if_action; \
	} else { \
	else_action; \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Log XML changes line-by-line in a formatted fashion
	*
	* \param[in] level Priority at which to log the messages
	* \param[in] xml XML to log
	*
	* \note This does nothing when \p level is \c LOG_STDOUT.
	*/
	#define pcmk__log_xml_changes(level, xml) do { \
	uint8_t _level = pcmk__clip_log_level(level); \
	static struct qb_log_callsite *xml_cs = NULL; \
	\
	switch (_level) { \
	case LOG_STDOUT: \
	case LOG_NEVER: \
	break; \
	default: \
	if (xml_cs == NULL) { \
	xml_cs = qb_log_callsite_get(__func__, __FILE__, \
	"xml-changes", _level, \
	__LINE__, 0); \
	} \
	if (crm_is_callsite_active(xml_cs, _level, 0)) { \
	pcmk__log_xml_changes_as(__FILE__, __func__, __LINE__, \
	0, _level, xml); \
	} \
	break; \
	} \
	} while(0)

	/*!
	* \internal
	* \brief Log an XML patchset line-by-line in a formatted fashion
	*
	* \param[in] level Priority at which to log the messages
	* \param[in] patchset XML patchset to log
	*
	* \note This does nothing when \p level is \c LOG_STDOUT.
	*/
	#define pcmk__log_xml_patchset(level, patchset) do { \
	uint8_t _level = pcmk__clip_log_level(level); \
	static struct qb_log_callsite *xml_cs = NULL; \
	\
	switch (_level) { \
	case LOG_STDOUT: \
	case LOG_NEVER: \
	break; \
	default: \
	if (xml_cs == NULL) { \
	xml_cs = qb_log_callsite_get(__func__, __FILE__, \
	"xml-patchset", _level, \
	__LINE__, 0); \
	} \
	if (crm_is_callsite_active(xml_cs, _level, 0)) { \
	pcmk__log_xml_patchset_as(__FILE__, __func__, __LINE__, \
	0, _level, patchset); \
	} \
	break; \
	} \
	} while(0)

	void pcmk__log_xml_changes_as(const char file, const char function,
	uint32_t line, uint32_t tags, uint8_t level,
	const xmlNode *xml);

	void pcmk__log_xml_patchset_as(const char file, const char function,
	uint32_t line, uint32_t tags, uint8_t level,
	const xmlNode *patchset);

	/*!
	* \internal
	* \brief Initialize logging for command line tools
	*
	* \param[in] name The name of the program
	* \param[in] verbosity How verbose to be in logging
	*
	* \note \p verbosity is not the same as the logging level (LOG_ERR, etc.).
	*/
	void pcmk__cli_init_logging(const char *name, unsigned int verbosity);

	int pcmk__add_logfile(const char *filename);
	void pcmk__add_logfiles(gchar *log_files, pcmk__output_t out);

	void pcmk__free_common_logger(void);

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_LOGGING_INTERNAL__H
	diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
	index 1e8086ca7a..39441508f3 100644
	--- a/include/crm/common/options_internal.h
	+++ b/include/crm/common/options_internal.h
	@@ -1,256 +1,247 @@
	/*
	* Copyright 2006-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_OPTIONS_INTERNAL__H
	#define PCMK__CRM_COMMON_OPTIONS_INTERNAL__H

	#ifndef PCMK__CONFIG_H
	#define PCMK__CONFIG_H
	#include <config.h> // _Noreturn
	#endif

	#include <glib.h> // GHashTable
	#include <stdbool.h> // bool

	#include <crm/common/strings.h> // pcmk_parse_interval_spec()
	#include <crm/common/output_internal.h> // pcmk__output_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	_Noreturn void pcmk__cli_help(char cmd);


	/*
	* Environment variable option handling
	*/

	const char pcmk__env_option(const char option);
	void pcmk__set_env_option(const char option, const char value, bool compat);
	bool pcmk__env_option_enabled(const char daemon, const char option);


	/*
	* Cluster option handling
	*/

	/*!
	* \internal
	* \enum pcmk__opt_flags
	* \brief Option flags
	*/
	enum pcmk__opt_flags {
	pcmk__opt_none = 0U, //!< No additional information

	/*!
	* \brief In CIB manager metadata
	*
	* \deprecated This flag will be removed with CIB manager metadata
	*/
	pcmk__opt_based = (1U << 0),

	/*!
	* \brief In controller metadata
	*
	* \deprecated This flag will be removed with controller metadata
	*/
	pcmk__opt_controld = (1U << 1),

	/*!
	* \brief In scheduler metadata
	*
	* \deprecated This flag will be removed with scheduler metadata
	*/
	pcmk__opt_schedulerd = (1U << 2),

	pcmk__opt_advanced = (1U << 3), //!< Advanced use only
	pcmk__opt_generated = (1U << 4), //!< Generated by Pacemaker
	pcmk__opt_deprecated = (1U << 5), //!< Option is deprecated
	pcmk__opt_fencing = (1U << 6), //!< Common fencing resource parameter
	pcmk__opt_primitive = (1U << 7), //!< Primitive resource meta-attribute
	};

	typedef struct pcmk__cluster_option_s {
	const char *name;
	const char *alt_name;
	const char *type;
	const char *values;
	const char *default_value;

	bool (is_valid)(const char );

	uint32_t flags; //!< Group of <tt>enum pcmk__opt_flags</tt>

	const char *description_short;
	const char *description_long;

	} pcmk__cluster_option_t;

	const char pcmk__cluster_option(GHashTable options, const char *name);

	int pcmk__output_cluster_options(pcmk__output_t out, const char name,
	const char desc_short, const char desc_long,
	uint32_t filter, bool all);
	int pcmk__output_fencing_params(pcmk__output_t out, const char name,
	const char desc_short, const char desc_long,
	bool all);
	int pcmk__output_primitive_meta(pcmk__output_t out, const char name,
	const char desc_short, const char desc_long,
	bool all);

	int pcmk__daemon_metadata(pcmk__output_t out, const char name,
	const char short_desc, const char long_desc,
	enum pcmk__opt_flags filter);

	void pcmk__validate_cluster_options(GHashTable *options);

	bool pcmk__valid_interval_spec(const char *value);
	bool pcmk__valid_boolean(const char *value);
	bool pcmk__valid_int(const char *value);
	bool pcmk__valid_positive_int(const char *value);
	bool pcmk__valid_no_quorum_policy(const char *value);
	bool pcmk__valid_percentage(const char *value);
	bool pcmk__valid_placement_strategy(const char *value);

	// from watchdog.c
	long pcmk__get_sbd_watchdog_timeout(void);
	bool pcmk__get_sbd_sync_resource_startup(void);
	long pcmk__auto_stonith_watchdog_timeout(void);
	bool pcmk__valid_stonith_watchdog_timeout(const char *value);

	// Constants for environment variable names
	#define PCMK__ENV_AUTHKEY_LOCATION "authkey_location"
	#define PCMK__ENV_BLACKBOX "blackbox"
	#define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled"
	#define PCMK__ENV_CLUSTER_TYPE "cluster_type"
	#define PCMK__ENV_DEBUG "debug"
	#define PCMK__ENV_DH_MAX_BITS "dh_max_bits"
	#define PCMK__ENV_FAIL_FAST "fail_fast"
	#define PCMK__ENV_IPC_BUFFER "ipc_buffer"
	#define PCMK__ENV_IPC_TYPE "ipc_type"
	#define PCMK__ENV_LOGFACILITY "logfacility"
	#define PCMK__ENV_LOGFILE "logfile"
	#define PCMK__ENV_LOGFILE_MODE "logfile_mode"
	#define PCMK__ENV_LOGPRIORITY "logpriority"
	#define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit"
	#define PCMK__ENV_NODE_START_STATE "node_start_state"
	#define PCMK__ENV_PANIC_ACTION "panic_action"
	#define PCMK__ENV_REMOTE_ADDRESS "remote_address"
	#define PCMK__ENV_REMOTE_SCHEMA_DIRECTORY "remote_schema_directory"
	#define PCMK__ENV_REMOTE_PID1 "remote_pid1"
	#define PCMK__ENV_REMOTE_PORT "remote_port"
	#define PCMK__ENV_RESPAWNED "respawned"
	#define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory"
	#define PCMK__ENV_SERVICE "service"
	#define PCMK__ENV_STDERR "stderr"
	#define PCMK__ENV_TLS_PRIORITIES "tls_priorities"
	#define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox"
	#define PCMK__ENV_TRACE_FILES "trace_files"
	#define PCMK__ENV_TRACE_FORMATS "trace_formats"
	#define PCMK__ENV_TRACE_FUNCTIONS "trace_functions"
	#define PCMK__ENV_TRACE_TAGS "trace_tags"
	#define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled"

	// Constants for meta-attribute names
	#define PCMK__META_CLONE "clone"
	#define PCMK__META_CONTAINER "container"
	#define PCMK__META_DIGESTS_ALL "digests-all"
	#define PCMK__META_DIGESTS_SECURE "digests-secure"
	#define PCMK__META_INTERNAL_RSC "internal_rsc"
	#define PCMK__META_MIGRATE_SOURCE "migrate_source"
	#define PCMK__META_MIGRATE_TARGET "migrate_target"
	#define PCMK__META_ON_NODE "on_node"
	#define PCMK__META_ON_NODE_UUID "on_node_uuid"
	#define PCMK__META_OP_NO_WAIT "op_no_wait"
	#define PCMK__META_OP_TARGET_RC "op_target_rc"
	#define PCMK__META_PHYSICAL_HOST "physical-host"
	#define PCMK__META_STONITH_ACTION "stonith_action"

	/* @TODO Plug these in. Currently, they're never set. These are op attrs for use
	* with https://projects.clusterlabs.org/T382.
	*/
	#define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op"
	#define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval"

	-// @COMPAT Deprecated meta-attribute since 2.1.0
	-#define PCMK__META_CAN_FAIL "can_fail"
	-
	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0
	#define PCMK__META_PROMOTED_MAX_LEGACY "master-max"

	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0
	#define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max"

	-// @COMPAT Deprecated meta-attribute since 2.0.0
	-#define PCMK__META_RESTART_TYPE "restart-type"
	-
	-// @COMPAT Deprecated meta-attribute since 2.0.0
	-#define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure"
	-
	// Constants for enumerated values
	#define PCMK__VALUE_ATTRD "attrd"
	#define PCMK__VALUE_BOLD "bold"
	#define PCMK__VALUE_BROADCAST "broadcast"
	#define PCMK__VALUE_CIB "cib"
	#define PCMK__VALUE_CIB_DIFF_NOTIFY "cib_diff_notify"
	#define PCMK__VALUE_CIB_NOTIFY "cib_notify"
	#define PCMK__VALUE_CIB_POST_NOTIFY "cib_post_notify"
	#define PCMK__VALUE_CIB_PRE_NOTIFY "cib_pre_notify"
	#define PCMK__VALUE_CIB_UPDATE_CONFIRMATION "cib_update_confirmation"
	#define PCMK__VALUE_CLUSTER "cluster"
	#define PCMK__VALUE_CRMD "crmd"
	#define PCMK__VALUE_EN "en"
	#define PCMK__VALUE_EPOCH "epoch"
	#define PCMK__VALUE_HEALTH_RED "health_red"
	#define PCMK__VALUE_HEALTH_YELLOW "health_yellow"
	#define PCMK__VALUE_INIT "init"
	#define PCMK__VALUE_LOCAL "local"
	#define PCMK__VALUE_LOST "lost"
	#define PCMK__VALUE_LRMD "lrmd"
	#define PCMK__VALUE_MAINT "maint"
	#define PCMK__VALUE_OUTPUT "output"
	#define PCMK__VALUE_PASSWORD "password"
	#define PCMK__VALUE_PRIMITIVE "primitive"
	#define PCMK__VALUE_REFRESH "refresh"
	#define PCMK__VALUE_REQUEST "request"
	#define PCMK__VALUE_RESPONSE "response"
	#define PCMK__VALUE_RSC_FAILED "rsc-failed"
	#define PCMK__VALUE_RSC_FAILURE_IGNORED "rsc-failure-ignored"
	#define PCMK__VALUE_RSC_MANAGED "rsc-managed"
	#define PCMK__VALUE_RSC_MULTIPLE "rsc-multiple"
	#define PCMK__VALUE_RSC_OK "rsc-ok"
	#define PCMK__VALUE_RUNNING "running"
	#define PCMK__VALUE_SCHEDULER "scheduler"
	#define PCMK__VALUE_SHUTDOWN_COMPLETE "shutdown_complete"
	#define PCMK__VALUE_SHUTTING_DOWN "shutting_down"
	#define PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE "st-async-timeout-value"
	#define PCMK__VALUE_ST_NOTIFY "st_notify"
	#define PCMK__VALUE_ST_NOTIFY_DISCONNECT "st_notify_disconnect"
	#define PCMK__VALUE_ST_NOTIFY_FENCE "st_notify_fence"
	#define PCMK__VALUE_ST_NOTIFY_HISTORY "st_notify_history"
	#define PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED "st_notify_history_synced"
	#define PCMK__VALUE_STARTING_DAEMONS "starting_daemons"
	#define PCMK__VALUE_STONITH_NG "stonith-ng"
	#define PCMK__VALUE_WAIT_FOR_PING "wait_for_ping"
	#define PCMK__VALUE_WARNING "warning"

	/* @COMPAT Deprecated since 2.1.7 (used with PCMK__XA_ORDERING attribute of
	* resource sets)
	*/
	#define PCMK__VALUE_GROUP "group"

	// @COMPAT Drop when daemon metadata commands are dropped
	#define PCMK__VALUE_TIME "time"

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__OPTIONS_INTERNAL__H
	diff --git a/include/crm/common/resources_internal.h b/include/crm/common/resources_internal.h
	index 2b5c40d22d..b499a808a7 100644
	--- a/include/crm/common/resources_internal.h
	+++ b/include/crm/common/resources_internal.h
	@@ -1,469 +1,462 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_RESOURCES_INTERNAL__H
	#define PCMK__CRM_COMMON_RESOURCES_INTERNAL__H

	#include <stdint.h> // uint32_t
	#include <glib.h> // gboolean, guint, GHashTable, GList
	#include <libxml/tree.h> // xmlNode

	#include <crm/common/resources.h> // pcmk_resource_t
	#include <crm/common/roles.h> // enum rsc_role_e
	#include <crm/common/scheduler_types.h> // pcmk_node_t, etc.

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \internal
	* \brief Set resource flags
	*
	* \param[in,out] resource Resource to set flags for
	* \param[in] flags_to_set Group of enum pcmk_rsc_flags to set
	*/
	#define pcmk__set_rsc_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	/*!
	* \internal
	* \brief Clear resource flags
	*
	* \param[in,out] resource Resource to clear flags for
	* \param[in] flags_to_clear Group of enum pcmk_rsc_flags to clear
	*/
	#define pcmk__clear_rsc_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	//! Resource variants supported by Pacemaker
	enum pcmk__rsc_variant {
	// Order matters: some code compares greater or lesser than
	pcmk__rsc_variant_unknown = -1, //!< Unknown resource variant
	pcmk__rsc_variant_primitive = 0, //!< Primitive resource
	pcmk__rsc_variant_group = 1, //!< Group resource
	pcmk__rsc_variant_clone = 2, //!< Clone resource
	pcmk__rsc_variant_bundle = 3, //!< Bundle resource
	};

	//! How to recover a resource that is incorrectly active on multiple nodes
	enum pcmk__multiply_active {
	pcmk__multiply_active_restart, //!< Stop on all, start on desired
	pcmk__multiply_active_stop, //!< Stop on all and leave stopped
	pcmk__multiply_active_block, //!< Do nothing to resource
	pcmk__multiply_active_unexpected, //!< Stop unexpected instances
	};

	-//! \deprecated
	-enum pcmk__restart {
	- pcmk__restart_restart,
	- pcmk__restart_ignore,
	-};
	-
	//! Resource scheduling flags
	enum pcmk__rsc_flags {
	// No resource flags set (compare with equality rather than bit set)
	pcmk__no_rsc_flags = 0ULL,

	// Whether resource has been removed from the configuration
	pcmk__rsc_removed = (1ULL << 0),

	/* NOTE: sbd (at least as of 1.5.2) uses pe_rsc_managed which equates to
	* this value, so the value should not be changed
	*/
	// Whether resource is managed
	pcmk__rsc_managed = (1ULL << 1),

	// Whether resource is blocked from further action
	pcmk__rsc_blocked = (1ULL << 2),

	// Whether resource has been removed but was launched
	pcmk__rsc_removed_launched = (1ULL << 3),

	// Whether resource has clone notifications enabled
	pcmk__rsc_notify = (1ULL << 4),

	// Whether resource is not an anonymous clone instance
	pcmk__rsc_unique = (1ULL << 5),

	// Whether resource's class is "stonith"
	pcmk__rsc_fence_device = (1ULL << 6),

	// Whether resource can be promoted and demoted
	pcmk__rsc_promotable = (1ULL << 7),

	// Whether resource has not yet been assigned to a node
	pcmk__rsc_unassigned = (1ULL << 8),

	// Whether resource is in the process of being assigned to a node
	pcmk__rsc_assigning = (1ULL << 9),

	// Whether resource is in the process of modifying allowed node scores
	pcmk__rsc_updating_nodes = (1ULL << 10),

	// Whether resource is in the process of scheduling actions to restart
	pcmk__rsc_restarting = (1ULL << 11),

	// Whether resource must be stopped (instead of demoted) if it is failed
	pcmk__rsc_stop_if_failed = (1ULL << 12),

	// Whether a reload action has been scheduled for resource
	pcmk__rsc_reload = (1ULL << 13),

	// Whether resource is a remote connection allowed to run on a remote node
	pcmk__rsc_remote_nesting_allowed = (1ULL << 14),

	// Whether resource has \c PCMK_META_CRITICAL meta-attribute enabled
	pcmk__rsc_critical = (1ULL << 15),

	// Whether resource is considered failed
	pcmk__rsc_failed = (1ULL << 16),

	// Flag for non-scheduler code to use to detect recursion loops
	pcmk__rsc_detect_loop = (1ULL << 17),

	// Whether resource is a Pacemaker Remote connection
	pcmk__rsc_is_remote_connection = (1ULL << 18),

	// Whether resource has pending start action in history
	pcmk__rsc_start_pending = (1ULL << 19),

	// Whether resource is probed only on nodes marked exclusive
	pcmk__rsc_exclusive_probes = (1ULL << 20),

	/*
	* Whether resource is multiply active with recovery set to
	* \c PCMK_VALUE_STOP_UNEXPECTED
	*/
	pcmk__rsc_stop_unexpected = (1ULL << 22),

	// Whether resource is allowed to live-migrate
	pcmk__rsc_migratable = (1ULL << 23),

	// Whether resource has an ignorable failure
	pcmk__rsc_ignore_failure = (1ULL << 24),

	// Whether resource is an implicit container resource for a bundle replica
	pcmk__rsc_replica_container = (1ULL << 25),

	// Whether resource, its node, or entire cluster is in maintenance mode
	pcmk__rsc_maintenance = (1ULL << 26),

	// Whether resource can be started or promoted only on quorate nodes
	pcmk__rsc_needs_quorum = (1ULL << 28),

	// Whether resource requires fencing before recovery if on unclean node
	pcmk__rsc_needs_fencing = (1ULL << 29),

	// Whether resource can be started or promoted only on unfenced nodes
	pcmk__rsc_needs_unfencing = (1ULL << 30),
	};

	// Where to look for a resource
	enum pcmk__rsc_node {
	pcmk__rsc_node_none = 0U, // Nowhere
	pcmk__rsc_node_assigned = (1U << 0), // Where resource is assigned
	pcmk__rsc_node_current = (1U << 1), // Where resource is running
	pcmk__rsc_node_pending = (1U << 2), // Where resource is pending
	};

	//! Resource assignment methods (implementation defined by libpacemaker)
	typedef struct pcmk__assignment_methods pcmk__assignment_methods_t;

	//! Resource object methods
	typedef struct {
	/*!
	* \internal
	* \brief Parse variant-specific resource XML from CIB into struct members
	*
	* \param[in,out] rsc Partially unpacked resource
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE if resource was unpacked successfully, otherwise FALSE
	*/
	gboolean (unpack)(pcmk_resource_t rsc, pcmk_scheduler_t *scheduler);

	/*!
	* \internal
	* \brief Search for a resource ID in a resource and its children
	*
	* \param[in] rsc Search this resource and its children
	* \param[in] id Search for this resource ID
	* \param[in] on_node If not NULL, limit search to resources on this node
	* \param[in] flags Group of enum pe_find flags
	*
	* \return Resource that matches search criteria if any, otherwise NULL
	*/
	pcmk_resource_t (find_rsc)(pcmk_resource_t rsc, const char search,
	const pcmk_node_t *node, int flags);

	/*!
	* \internal
	* \brief Get value of a resource instance attribute
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to use to evaluate rules
	* \param[in] create Ignored
	* \param[in] name Name of instance attribute to check
	* \param[in,out] scheduler Scheduler data
	*
	* \return Value of requested attribute if available, otherwise NULL
	* \note The caller is responsible for freeing the result using free().
	*/
	char (parameter)(pcmk_resource_t rsc, pcmk_node_t node, gboolean create,
	const char name, pcmk_scheduler_t scheduler);

	/*!
	* \internal
	* \brief Check whether a resource is active
	*
	* \param[in] rsc Resource to check
	* \param[in] all If \p rsc is collective, all instances must be active
	*
	* \return TRUE if \p rsc is active, otherwise FALSE
	*/
	gboolean (active)(pcmk_resource_t rsc, gboolean all);

	/*!
	* \internal
	* \brief Get resource's current or assigned role
	*
	* \param[in] rsc Resource to check
	* \param[in] current If TRUE, check current role, otherwise assigned role
	*
	* \return Current or assigned role of \p rsc
	*/
	enum rsc_role_e (state)(const pcmk_resource_t rsc, gboolean current);

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] target Which resource conditions to target (group of
	* enum pcmk__rsc_node flags)
	*
	* \return If list contains only one node, that node, otherwise NULL
	*/
	pcmk_node_t (location)(const pcmk_resource_t rsc, GList *list,
	uint32_t target);

	/*!
	* \internal
	* \brief Free all memory used by a resource
	*
	* \param[in,out] rsc Resource to free
	*/
	void (free)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Increment cluster's instance counts for a resource
	*
	* Given a resource, increment its cluster's ninstances, disabled_resources,
	* and blocked_resources counts for the resource and its descendants.
	*
	* \param[in,out] rsc Resource to count
	*/
	void (count)(pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Check whether a given resource is in a list of resources
	*
	* \param[in] rsc Resource ID to check for
	* \param[in] only_rsc List of resource IDs to check
	* \param[in] check_parent If TRUE, check top ancestor as well
	*
	* \return TRUE if \p rsc, its top parent if requested, or '*' is in
	* \p only_rsc, otherwise FALSE
	*/
	gboolean (is_filtered)(const pcmk_resource_t rsc, GList *only_rsc,
	gboolean check_parent);

	/*!
	* \internal
	* \brief Find a node (and optionally count all) where resource is active
	*
	* \param[in] rsc Resource to check
	* \param[out] count_all If not NULL, set this to count of active nodes
	* \param[out] count_clean If not NULL, set this to count of clean nodes
	*
	* \return A node where the resource is active, preferring the source node
	* if the resource is involved in a partial migration, or a clean,
	* online node if the resource's \c PCMK_META_REQUIRES is
	* \c PCMK_VALUE_QUORUM or \c PCMK_VALUE_NOTHING, otherwise \c NULL.
	*/
	pcmk_node_t (active_node)(const pcmk_resource_t *rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	/*!
	* \internal
	* \brief Get maximum resource instances per node
	*
	* \param[in] rsc Resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int (max_per_node)(const pcmk_resource_t rsc);
	} pcmk__rsc_methods_t;

	// Implementation of pcmk__resource_private_t
	struct pcmk__resource_private {
	enum pcmk__rsc_variant variant; // Resource variant
	void *variant_opaque; // Variant-specific data
	char *history_id; // Resource instance ID in history
	GHashTable *meta; // Resource meta-attributes
	GHashTable *utilization; // Resource utilization attributes
	int priority; // Priority relative other resources
	int promotion_priority; // Promotion priority on assigned node
	enum rsc_role_e orig_role; // Resource's role at start of transition
	enum rsc_role_e next_role; // Resource's role at end of transition
	int stickiness; // Extra preference for current node
	guint failure_expiration_ms; // Failures expire after this much time
	int ban_after_failures; // Ban from node after this many failures
	guint remote_reconnect_ms; // Retry interval for remote connections
	char *pending_action; // Pending action in history, if any
	const pcmk_node_t *pending_node;// Node on which pending_action is happening
	time_t lock_time; // When shutdown lock started
	const pcmk_node_t *lock_node; // Node that resource is shutdown-locked to
	GList *actions; // Actions scheduled for resource
	GList *children; // Resource's child resources, if any
	pcmk_resource_t *parent; // Resource's parent resource, if any
	pcmk_scheduler_t *scheduler; // Scheduler data containing resource
	- enum pcmk__restart restart_type; // Deprecated

	// Resource configuration (possibly expanded from template)
	xmlNode *xml;

	// Original resource configuration, if using template
	xmlNode *orig_xml;

	// Configuration of resource operations (possibly expanded from template)
	xmlNode *ops_xml;

	/*
	* Resource parameters may have node-attribute-based rules, which means the
	* values can vary by node. This table has node names as keys and parameter
	* name/value tables as values. Use pe_rsc_params() to get the table for a
	* given node rather than use this directly.
	*/
	GHashTable *parameter_cache;

	/* A "launcher" is defined in one of these ways:
	*
	* - A Pacemaker Remote connection for a guest node or bundle node has its
	* launcher set to the resource that starts the guest or the bundle
	* replica's container.
	*
	* - If the user configures the PCMK__META_CONTAINER meta-attribute for this
	* resource, the launcher is set to that.
	*
	* If the launcher is a Pacemaker Remote connection resource, this
	* resource may run only on the node created by that connection.
	*
	* Otherwise, this resource will be colocated with and ordered after the
	* launcher, and failures of this resource will cause the launcher to be
	* recovered instead of this one. This is appropriate for monitoring-only
	* resources that represent a service launched by the other resource.
	*/
	pcmk_resource_t *launcher;

	// Resources launched by this one, if any (pcmk_resource_t *)
	GList *launched;

	// What to do if the resource is incorrectly active on multiple nodes
	enum pcmk__multiply_active multiply_active_policy;

	/* The assigned node (if not NULL) is the one where the resource should
	* be active by the end of the current scheduler transition. Only primitive
	* resources have an assigned node.
	*
	* @TODO This should probably be part of the primitive variant data.
	*/
	pcmk_node_t *assigned_node;

	/* The active nodes are ones where the resource is (or might be, if
	* insufficient information is available to be sure) already active at the
	* start of the current scheduler transition.
	*
	* For primitive resources, there should be at most one, but could be more
	* if it is (incorrectly) multiply active. For collective resources, this
	* combines active nodes of all descendants.
	*/
	GList *active_nodes;

	// Nodes where resource has been probed (key is node ID, not name)
	GHashTable *probed_nodes;

	// Nodes where resource is allowed to run (key is node ID, not name)
	GHashTable *allowed_nodes;

	// The source node, if migrate_to completed but migrate_from has not
	pcmk_node_t *partial_migration_source;

	// The destination node, if migrate_to completed but migrate_from has not
	pcmk_node_t *partial_migration_target;

	// Source nodes where stop is needed after migrate_from and migrate_to
	GList *dangling_migration_sources;

	/* Pay special attention to whether you want to use with_this_colocations
	* and this_with_colocations directly, which include only colocations
	* explicitly involving this resource, or call libpacemaker's
	* pcmk__with_this_colocations() and pcmk__this_with_colocations()
	* functions, which may return relevant colocations involving the resource's
	* ancestors as well.
	*/

	// Colocations of other resources with this one
	GList *with_this_colocations;

	// Colocations of this resource with others
	GList *this_with_colocations;

	GList *location_constraints; // Location constraints for resource
	GList *ticket_constraints; // Ticket constraints for resource

	const pcmk__rsc_methods_t *fns; // Resource object methods
	const pcmk__assignment_methods_t *cmds; // Resource assignment methods
	};

	const char pcmk__multiply_active_text(const pcmk_resource_t rsc);

	/*!
	* \internal
	* \brief Get node where resource is currently active (if any)
	*
	* \param[in] rsc Resource to check
	*
	* \return Node that \p rsc is active on, if any, otherwise NULL
	*/
	static inline pcmk_node_t *
	pcmk__current_node(const pcmk_resource_t *rsc)
	{
	if (rsc == NULL) {
	return NULL;
	}
	return rsc->priv->fns->active_node(rsc, NULL, NULL);
	}

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_RESOURCES_INTERNAL__H
	diff --git a/include/pcmki/pcmki_transition.h b/include/pcmki/pcmki_transition.h
	index d10fe83393..d66d7e43ce 100644
	--- a/include/pcmki/pcmki_transition.h
	+++ b/include/pcmki/pcmki_transition.h
	@@ -1,184 +1,181 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__PCMKI_PCMKI_TRANSITION__H
	#define PCMK__PCMKI_PCMKI_TRANSITION__H

	#include <stdbool.h> // bool
	#include <stdint.h> // uint32_t
	#include <sys/types.h> // time_t
	#include <glib.h> // guint, GList, GHashTable
	#include <libxml/tree.h> // xmlNode

	#include <crm/common/scheduler_types.h> // pcmk_scheduler_t
	#include <crm/lrmd_events.h> // lrmd_event_data_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	enum pcmk__graph_action_type {
	pcmk__pseudo_graph_action,
	pcmk__rsc_graph_action,
	pcmk__cluster_graph_action,
	};

	enum pcmk__synapse_flags {
	pcmk__synapse_ready = (1 << 0),
	pcmk__synapse_failed = (1 << 1),
	pcmk__synapse_executed = (1 << 2),
	pcmk__synapse_confirmed = (1 << 3),
	};

	typedef struct {
	int id;
	int priority;

	uint32_t flags; // Group of pcmk__synapse_flags

	GList actions; / pcmk__graph_action_t* */
	GList inputs; / pcmk__graph_action_t* */
	} pcmk__graph_synapse_t;

	#define pcmk__set_synapse_flags(synapse, flags_to_set) do { \
	(synapse)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Synapse", "synapse", \
	(synapse)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pcmk__clear_synapse_flags(synapse, flags_to_clear) do { \
	(synapse)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Synapse", "synapse", \
	(synapse)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	enum pcmk__graph_action_flags {
	pcmk__graph_action_sent_update = (1 << 0), /* sent to the CIB */
	pcmk__graph_action_executed = (1 << 1), /* sent to the CRM */
	pcmk__graph_action_confirmed = (1 << 2),
	pcmk__graph_action_failed = (1 << 3),
	-
	- //! \deprecated Will be removed in a future release
	- pcmk__graph_action_can_fail = (1 << 4),
	};

	typedef struct {
	int id;
	int timeout;
	int timer;
	guint interval_ms;
	GHashTable *params;
	enum pcmk__graph_action_type type;
	pcmk__graph_synapse_t *synapse;

	uint32_t flags; // Group of pcmk__graph_action_flags

	xmlNode *xml;

	} pcmk__graph_action_t;

	#define pcmk__set_graph_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", "action", \
	(action)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pcmk__clear_graph_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", "action", \
	(action)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	// What to do after finished processing a transition graph
	enum pcmk__graph_next {
	// Order matters: lowest priority to highest
	pcmk__graph_done, // Transition complete, nothing further needed
	pcmk__graph_wait, // Transition interrupted, wait for further changes
	pcmk__graph_restart, // Transition interrupted, start a new one
	pcmk__graph_shutdown, // Transition interrupted, local shutdown needed
	};

	typedef struct {
	int id;
	char *source;
	int abort_priority;

	bool complete;
	const char *abort_reason;
	enum pcmk__graph_next completion_action;

	int num_actions;
	int num_synapses;

	int batch_limit;
	guint network_delay;
	guint stonith_timeout;

	int fired;
	int pending;
	int skipped;
	int completed;
	int incomplete;

	GList synapses; / pcmk__graph_synapse_t* */

	int migration_limit;

	//! Failcount after one failed stop action
	char *failed_stop_offset;

	//! Failcount after one failed start action
	char *failed_start_offset;

	//! Time (from epoch) by which the controller should re-run the scheduler
	time_t recheck_by;
	} pcmk__graph_t;


	typedef struct {
	int (pseudo) (pcmk__graph_t graph, pcmk__graph_action_t *action);
	int (rsc) (pcmk__graph_t graph, pcmk__graph_action_t *action);
	int (cluster) (pcmk__graph_t graph, pcmk__graph_action_t *action);
	int (fence) (pcmk__graph_t graph, pcmk__graph_action_t *action);
	bool (allowed) (pcmk__graph_t graph, pcmk__graph_action_t *action);
	} pcmk__graph_functions_t;

	enum pcmk__graph_status {
	pcmk__graph_active, // Some actions have been performed
	pcmk__graph_pending, // No actions performed yet
	pcmk__graph_complete,
	pcmk__graph_terminated,
	};

	void pcmk__set_graph_functions(pcmk__graph_functions_t *fns);
	pcmk__graph_t pcmk__unpack_graph(const xmlNode xml_graph,
	const char *reference);
	enum pcmk__graph_status pcmk__execute_graph(pcmk__graph_t *graph);
	void pcmk__update_graph(pcmk__graph_t *graph,
	const pcmk__graph_action_t *action);
	void pcmk__free_graph(pcmk__graph_t *graph);
	const char *pcmk__graph_status2text(enum pcmk__graph_status state);
	void pcmk__log_graph(unsigned int log_level, pcmk__graph_t *graph);
	void pcmk__log_graph_action(int log_level, pcmk__graph_action_t *action);
	void pcmk__log_transition_summary(const pcmk_scheduler_t *scheduler,
	const char *filename);
	lrmd_event_data_t pcmk__event_from_graph_action(const xmlNode resource,
	const pcmk__graph_action_t *action,
	int status, int rc,
	const char *exit_reason);

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__PCMKI_PCMKI_TRANSITION__H
	diff --git a/lib/pacemaker/pcmk_graph_consumer.c b/lib/pacemaker/pcmk_graph_consumer.c
	index 851d73b829..9cd0897213 100644
	--- a/lib/pacemaker/pcmk_graph_consumer.c
	+++ b/lib/pacemaker/pcmk_graph_consumer.c
	@@ -1,884 +1,865 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd_internal.h>
	#include <pacemaker-internal.h>


	/*
	* Functions for freeing transition graph objects
	*/

	/*!
	* \internal
	* \brief Free a transition graph action object
	*
	* \param[in,out] user_data Action to free
	*/
	static void
	free_graph_action(gpointer user_data)
	{
	pcmk__graph_action_t *action = user_data;

	if (action->timer != 0) {
	crm_warn("Cancelling timer for graph action %d", action->id);
	g_source_remove(action->timer);
	}
	if (action->params != NULL) {
	g_hash_table_destroy(action->params);
	}
	pcmk__xml_free(action->xml);
	free(action);
	}

	/*!
	* \internal
	* \brief Free a transition graph synapse object
	*
	* \param[in,out] user_data Synapse to free
	*/
	static void
	free_graph_synapse(gpointer user_data)
	{
	pcmk__graph_synapse_t *synapse = user_data;

	g_list_free_full(synapse->actions, free_graph_action);
	g_list_free_full(synapse->inputs, free_graph_action);
	free(synapse);
	}

	/*!
	* \internal
	* \brief Free a transition graph object
	*
	* \param[in,out] graph Transition graph to free
	*/
	void
	pcmk__free_graph(pcmk__graph_t *graph)
	{
	if (graph != NULL) {
	g_list_free_full(graph->synapses, free_graph_synapse);
	free(graph->source);
	free(graph->failed_stop_offset);
	free(graph->failed_start_offset);
	free(graph);
	}
	}


	/*
	* Functions for updating graph
	*/

	/*!
	* \internal
	* \brief Update synapse after completed prerequisite
	*
	* A synapse is ready to be executed once all its prerequisite actions (inputs)
	* complete. Given a completed action, check whether it is an input for a given
	* synapse, and if so, mark the input as confirmed, and mark the synapse as
	* ready if appropriate.
	*
	* \param[in,out] synapse Transition graph synapse to update
	* \param[in] action_id ID of an action that completed
	*
	* \note The only substantial effect here is confirming synapse inputs.
	* should_fire_synapse() will recalculate pcmk__synapse_ready, so the only
	* thing that uses the pcmk__synapse_ready from here is
	* synapse_state_str().
	*/
	static void
	update_synapse_ready(pcmk__graph_synapse_t *synapse, int action_id)
	{
	if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
	return; // All inputs have already been confirmed
	}

	// Presume ready until proven otherwise
	pcmk__set_synapse_flags(synapse, pcmk__synapse_ready);

	for (GList *lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_action_t prereq = (pcmk__graph_action_t ) lpc->data;

	if (prereq->id == action_id) {
	crm_trace("Confirming input %d of synapse %d",
	action_id, synapse->id);
	pcmk__set_graph_action_flags(prereq, pcmk__graph_action_confirmed);

	} else if (!pcmk_is_set(prereq->flags, pcmk__graph_action_confirmed)) {
	pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
	crm_trace("Synapse %d still not ready after action %d",
	synapse->id, action_id);
	}
	}
	if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
	crm_trace("Synapse %d is now ready to execute", synapse->id);
	}
	}

	/*!
	* \internal
	* \brief Update action and synapse confirmation after action completion
	*
	* \param[in,out] synapse Transition graph synapse that action belongs to
	* \param[in] action_id ID of action that completed
	*/
	static void
	update_synapse_confirmed(pcmk__graph_synapse_t *synapse, int action_id)
	{
	bool all_confirmed = true;

	for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_action_t action = (pcmk__graph_action_t ) lpc->data;

	if (action->id == action_id) {
	crm_trace("Confirmed action %d of synapse %d",
	action_id, synapse->id);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);

	} else if (all_confirmed &&
	!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	all_confirmed = false;
	crm_trace("Synapse %d still not confirmed after action %d",
	synapse->id, action_id);
	}
	}

	if (all_confirmed
	&& !pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
	crm_trace("Confirmed synapse %d", synapse->id);
	pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed);
	}
	}

	/*!
	* \internal
	* \brief Update the transition graph with a completed action result
	*
	* \param[in,out] graph Transition graph to update
	* \param[in] action Action that completed
	*/
	void
	pcmk__update_graph(pcmk__graph_t graph, const pcmk__graph_action_t action)
	{
	for (GList *lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_synapse_t synapse = (pcmk__graph_synapse_t ) lpc->data;

	if (pcmk_any_flags_set(synapse->flags,
	pcmk__synapse_confirmed\|pcmk__synapse_failed)) {
	continue; // This synapse already completed

	} else if (pcmk_is_set(synapse->flags, pcmk__synapse_executed)) {
	update_synapse_confirmed(synapse, action->id);

	} else if (!pcmk_is_set(action->flags, pcmk__graph_action_failed)
	\|\| (synapse->priority == PCMK_SCORE_INFINITY)) {
	update_synapse_ready(synapse, action->id);
	}
	}
	}


	/*
	* Functions for executing graph
	*/

	/* A transition graph consists of various types of actions. The library caller
	* registers execution functions for each action type, which will be stored
	* here.
	*/
	static pcmk__graph_functions_t *graph_fns = NULL;

	/*!
	* \internal
	* \brief Set transition graph execution functions
	*
	* \param[in] Execution functions to use
	*/
	void
	pcmk__set_graph_functions(pcmk__graph_functions_t *fns)
	{

	pcmk__assert((fns != NULL) && (fns->rsc != NULL) && (fns->cluster != NULL)
	&& (fns->pseudo != NULL) && (fns->fence != NULL));
	crm_debug("Setting custom functions for executing transition graphs");
	graph_fns = fns;
	}

	/*!
	* \internal
	* \brief Check whether a graph synapse is ready to be executed
	*
	* \param[in,out] graph Transition graph that synapse is part of
	* \param[in,out] synapse Synapse to check
	*
	* \return true if synapse is ready, false otherwise
	*/
	static bool
	should_fire_synapse(pcmk__graph_t graph, pcmk__graph_synapse_t synapse)
	{
	GList *lpc = NULL;

	pcmk__set_synapse_flags(synapse, pcmk__synapse_ready);
	for (lpc = synapse->inputs; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_action_t prereq = (pcmk__graph_action_t ) lpc->data;

	if (!(pcmk_is_set(prereq->flags, pcmk__graph_action_confirmed))) {
	crm_trace("Input %d for synapse %d not yet confirmed",
	prereq->id, synapse->id);
	pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
	break;

	- } else if (pcmk_is_set(prereq->flags, pcmk__graph_action_failed)
	- && !pcmk_is_set(prereq->flags,
	- pcmk__graph_action_can_fail)) {
	+ } else if (pcmk_is_set(prereq->flags, pcmk__graph_action_failed)) {
	crm_trace("Input %d for synapse %d confirmed but failed",
	prereq->id, synapse->id);
	pcmk__clear_synapse_flags(synapse, pcmk__synapse_ready);
	break;
	}
	}
	if (pcmk_is_set(synapse->flags, pcmk__synapse_ready)) {
	crm_trace("Synapse %d is ready to execute", synapse->id);
	} else {
	return false;
	}

	for (lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_action_t a = (pcmk__graph_action_t ) lpc->data;

	if (a->type == pcmk__pseudo_graph_action) {
	/* None of the below applies to pseudo ops */

	} else if (synapse->priority < graph->abort_priority) {
	crm_trace("Skipping synapse %d: priority %d is less than "
	"abort priority %d",
	synapse->id, synapse->priority, graph->abort_priority);
	graph->skipped++;
	return false;

	} else if (graph_fns->allowed && !(graph_fns->allowed(graph, a))) {
	crm_trace("Deferring synapse %d: not allowed", synapse->id);
	return false;
	}
	}

	return true;
	}

	/*!
	* \internal
	* \brief Initiate an action from a transition graph
	*
	* \param[in,out] graph Transition graph containing action
	* \param[in,out] action Action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	initiate_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *id = pcmk__xe_id(action->xml);

	CRM_CHECK(id != NULL, return EINVAL);
	CRM_CHECK(!pcmk_is_set(action->flags, pcmk__graph_action_executed),
	return pcmk_rc_already);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
	switch (action->type) {
	case pcmk__pseudo_graph_action:
	crm_trace("Executing pseudo-action %d (%s)", action->id, id);
	return graph_fns->pseudo(graph, action);

	case pcmk__rsc_graph_action:
	crm_trace("Executing resource action %d (%s)", action->id, id);
	return graph_fns->rsc(graph, action);

	case pcmk__cluster_graph_action:
	if (pcmk__str_eq(crm_element_value(action->xml, PCMK_XA_OPERATION),
	PCMK_ACTION_STONITH, pcmk__str_none)) {
	crm_trace("Executing fencing action %d (%s)",
	action->id, id);
	return graph_fns->fence(graph, action);
	}
	crm_trace("Executing cluster action %d (%s)", action->id, id);
	return graph_fns->cluster(graph, action);

	default:
	crm_err("Unsupported graph action type <%s " PCMK_XA_ID "='%s'> "
	"(bug?)",
	action->xml->name, id);
	return EINVAL;
	}
	}

	/*!
	* \internal
	* \brief Execute a graph synapse
	*
	* \param[in,out] graph Transition graph with synapse to execute
	* \param[in,out] synapse Synapse to execute
	*
	* \return Standard Pacemaker return value
	*/
	static int
	fire_synapse(pcmk__graph_t graph, pcmk__graph_synapse_t synapse)
	{
	pcmk__set_synapse_flags(synapse, pcmk__synapse_executed);
	for (GList *lpc = synapse->actions; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_action_t action = (pcmk__graph_action_t ) lpc->data;
	int rc = initiate_action(graph, action);

	if (rc != pcmk_rc_ok) {
	crm_err("Failed initiating <%s " PCMK_XA_ID "=%d> in synapse %d: "
	"%s",
	action->xml->name, action->id, synapse->id,
	pcmk_rc_str(rc));
	pcmk__set_synapse_flags(synapse, pcmk__synapse_confirmed);
	pcmk__set_graph_action_flags(action,
	pcmk__graph_action_confirmed
	\|pcmk__graph_action_failed);
	return pcmk_rc_error;
	}
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Dummy graph method that can be used with simulations
	*
	* \param[in,out] graph Transition graph containing action
	* \param[in,out] action Graph action to be initiated
	*
	* \return Standard Pacemaker return code
	* \note If the PE_fail environment variable is set to the action ID,
	* then the graph action will be marked as failed.
	*/
	static int
	pseudo_action_dummy(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	static int fail = -1;

	if (fail < 0) {
	long long fail_ll;

	if ((pcmk__scan_ll(getenv("PE_fail"), &fail_ll, 0LL) == pcmk_rc_ok)
	&& (fail_ll > 0LL) && (fail_ll <= INT_MAX)) {
	fail = (int) fail_ll;
	} else {
	fail = 0;
	}
	}

	if (action->id == fail) {
	crm_err("Dummy event handler: pretending action %d failed", action->id);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	graph->abort_priority = PCMK_SCORE_INFINITY;
	} else {
	crm_trace("Dummy event handler: action %d initiated", action->id);
	}
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	static pcmk__graph_functions_t default_fns = {
	pseudo_action_dummy,
	pseudo_action_dummy,
	pseudo_action_dummy,
	pseudo_action_dummy
	};

	/*!
	* \internal
	* \brief Execute all actions in a transition graph
	*
	* \param[in,out] graph Transition graph to execute
	*
	* \return Status of transition after execution
	*/
	enum pcmk__graph_status
	pcmk__execute_graph(pcmk__graph_t *graph)
	{
	GList *lpc = NULL;
	int log_level = LOG_DEBUG;
	enum pcmk__graph_status pass_result = pcmk__graph_active;
	const char *status = "In progress";

	if (graph_fns == NULL) {
	graph_fns = &default_fns;
	}
	if (graph == NULL) {
	return pcmk__graph_complete;
	}

	graph->fired = 0;
	graph->pending = 0;
	graph->skipped = 0;
	graph->completed = 0;
	graph->incomplete = 0;

	// Count completed and in-flight synapses
	for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_synapse_t synapse = (pcmk__graph_synapse_t ) lpc->data;

	if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
	graph->completed++;

	} else if (!pcmk_is_set(synapse->flags, pcmk__synapse_failed)
	&& pcmk_is_set(synapse->flags, pcmk__synapse_executed)) {
	graph->pending++;
	}
	}
	crm_trace("Executing graph %d (%d synapses already completed, %d pending)",
	graph->id, graph->completed, graph->pending);

	// Execute any synapses that are ready
	for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	pcmk__graph_synapse_t synapse = (pcmk__graph_synapse_t ) lpc->data;

	if ((graph->batch_limit > 0)
	&& (graph->pending >= graph->batch_limit)) {

	crm_debug("Throttling graph execution: batch limit (%d) reached",
	graph->batch_limit);
	break;

	} else if (pcmk_is_set(synapse->flags, pcmk__synapse_failed)) {
	graph->skipped++;
	continue;

	} else if (pcmk_any_flags_set(synapse->flags,
	pcmk__synapse_confirmed
	\|pcmk__synapse_executed)) {
	continue; // Already handled

	} else if (should_fire_synapse(graph, synapse)) {
	graph->fired++;
	if (fire_synapse(graph, synapse) != pcmk_rc_ok) {
	crm_err("Synapse %d failed to fire", synapse->id);
	log_level = LOG_ERR;
	graph->abort_priority = PCMK_SCORE_INFINITY;
	graph->incomplete++;
	graph->fired--;
	}

	if (!(pcmk_is_set(synapse->flags, pcmk__synapse_confirmed))) {
	graph->pending++;
	}

	} else {
	crm_trace("Synapse %d cannot fire", synapse->id);
	graph->incomplete++;
	}
	}

	if ((graph->pending == 0) && (graph->fired == 0)) {
	graph->complete = true;

	if ((graph->incomplete != 0) && (graph->abort_priority <= 0)) {
	log_level = LOG_WARNING;
	pass_result = pcmk__graph_terminated;
	status = "Terminated";

	} else if (graph->skipped != 0) {
	log_level = LOG_NOTICE;
	pass_result = pcmk__graph_complete;
	status = "Stopped";

	} else {
	log_level = LOG_NOTICE;
	pass_result = pcmk__graph_complete;
	status = "Complete";
	}

	} else if (graph->fired == 0) {
	pass_result = pcmk__graph_pending;
	}

	do_crm_log(log_level,
	"Transition %d (Complete=%d, Pending=%d,"
	" Fired=%d, Skipped=%d, Incomplete=%d, Source=%s): %s",
	graph->id, graph->completed, graph->pending, graph->fired,
	graph->skipped, graph->incomplete, graph->source, status);

	return pass_result;
	}


	/*
	* Functions for unpacking transition graph XML into structs
	*/

	/*!
	* \internal
	* \brief Unpack a transition graph action from XML
	*
	* \param[in] parent Synapse that action is part of
	* \param[in] xml_action Action XML to unparse
	*
	* \return Newly allocated action on success, or NULL otherwise
	*/
	static pcmk__graph_action_t *
	unpack_action(pcmk__graph_synapse_t parent, xmlNode xml_action)
	{
	enum pcmk__graph_action_type action_type;
	pcmk__graph_action_t *action = NULL;
	const char *value = pcmk__xe_id(xml_action);

	if (value == NULL) {
	crm_err("Ignoring transition graph action without " PCMK_XA_ID
	" (bug?)");
	crm_log_xml_trace(xml_action, "invalid");
	return NULL;
	}

	if (pcmk__xe_is(xml_action, PCMK__XE_RSC_OP)) {
	action_type = pcmk__rsc_graph_action;

	} else if (pcmk__xe_is(xml_action, PCMK__XE_PSEUDO_EVENT)) {
	action_type = pcmk__pseudo_graph_action;

	} else if (pcmk__xe_is(xml_action, PCMK__XE_CRM_EVENT)) {
	action_type = pcmk__cluster_graph_action;

	} else {
	crm_err("Ignoring transition graph action of unknown type '%s' (bug?)",
	xml_action->name);
	crm_log_xml_trace(xml_action, "invalid");
	return NULL;
	}

	action = calloc(1, sizeof(pcmk__graph_action_t));
	if (action == NULL) {
	crm_perror(LOG_CRIT, "Cannot unpack transition graph action");
	crm_log_xml_trace(xml_action, "lost");
	return NULL;
	}

	pcmk__scan_min_int(value, &(action->id), -1);
	action->type = pcmk__rsc_graph_action;
	action->xml = pcmk__xml_copy(NULL, xml_action);
	action->synapse = parent;
	action->type = action_type;
	action->params = xml2list(action->xml);

	value = crm_meta_value(action->params, PCMK_META_TIMEOUT);
	pcmk__scan_min_int(value, &(action->timeout), 0);

	/* Take PCMK_META_START_DELAY into account for the timeout of the action
	* timer
	*/
	value = crm_meta_value(action->params, PCMK_META_START_DELAY);
	{
	int start_delay;

	pcmk__scan_min_int(value, &start_delay, 0);
	action->timeout += start_delay;
	}

	if (pcmk__guint_from_hash(action->params, CRM_META "_" PCMK_META_INTERVAL,
	0, &(action->interval_ms)) != pcmk_rc_ok) {
	action->interval_ms = 0;
	}

	- value = crm_meta_value(action->params, PCMK__META_CAN_FAIL);
	- if (value != NULL) {
	- // @COMPAT Not possible with schema validation enabled
	- int can_fail = 0;
	-
	- if ((crm_str_to_boolean(value, &can_fail) > 0) && (can_fail > 0)) {
	- pcmk__set_graph_action_flags(action, pcmk__graph_action_can_fail);
	- } else {
	- pcmk__clear_graph_action_flags(action, pcmk__graph_action_can_fail);
	- }
	-
	- if (pcmk_is_set(action->flags, pcmk__graph_action_can_fail)) {
	- crm_warn("Support for the " PCMK__META_CAN_FAIL " meta-attribute "
	- "is deprecated and will be removed in a future release");
	- }
	- }
	-
	crm_trace("Action %d has timer set to %dms", action->id, action->timeout);

	return action;
	}

	/*!
	* \internal
	* \brief Unpack transition graph synapse from XML
	*
	* \param[in,out] new_graph Transition graph that synapse is part of
	* \param[in] xml_synapse Synapse XML
	*
	* \return Newly allocated synapse on success, or NULL otherwise
	*/
	static pcmk__graph_synapse_t *
	unpack_synapse(pcmk__graph_t new_graph, const xmlNode xml_synapse)
	{
	const char *value = NULL;
	xmlNode *action_set = NULL;
	pcmk__graph_synapse_t *new_synapse = NULL;

	crm_trace("Unpacking synapse %s", pcmk__xe_id(xml_synapse));

	new_synapse = calloc(1, sizeof(pcmk__graph_synapse_t));
	if (new_synapse == NULL) {
	return NULL;
	}

	pcmk__scan_min_int(pcmk__xe_id(xml_synapse), &(new_synapse->id), 0);

	value = crm_element_value(xml_synapse, PCMK__XA_PRIORITY);
	pcmk__scan_min_int(value, &(new_synapse->priority), 0);

	CRM_CHECK(new_synapse->id >= 0,
	free_graph_synapse((gpointer) new_synapse); return NULL);

	new_graph->num_synapses++;

	crm_trace("Unpacking synapse %s action sets",
	crm_element_value(xml_synapse, PCMK_XA_ID));

	for (action_set = pcmk__xe_first_child(xml_synapse, "action_set", NULL,
	NULL);
	action_set != NULL; action_set = pcmk__xe_next_same(action_set)) {

	for (xmlNode *action = pcmk__xe_first_child(action_set, NULL, NULL,
	NULL);
	action != NULL; action = pcmk__xe_next(action)) {

	pcmk__graph_action_t *new_action = unpack_action(new_synapse,
	action);

	if (new_action == NULL) {
	continue;
	}

	crm_trace("Adding action %d to synapse %d",
	new_action->id, new_synapse->id);
	new_graph->num_actions++;
	new_synapse->actions = g_list_append(new_synapse->actions,
	new_action);
	}
	}

	crm_trace("Unpacking synapse %s inputs", pcmk__xe_id(xml_synapse));

	for (xmlNode *inputs = pcmk__xe_first_child(xml_synapse, "inputs", NULL,
	NULL);
	inputs != NULL; inputs = pcmk__xe_next_same(inputs)) {

	for (xmlNode *trigger = pcmk__xe_first_child(inputs, "trigger", NULL,
	NULL);
	trigger != NULL; trigger = pcmk__xe_next_same(trigger)) {

	for (xmlNode *input = pcmk__xe_first_child(trigger, NULL, NULL,
	NULL);
	input != NULL; input = pcmk__xe_next(input)) {

	pcmk__graph_action_t *new_input = unpack_action(new_synapse,
	input);

	if (new_input == NULL) {
	continue;
	}

	crm_trace("Adding input %d to synapse %d",
	new_input->id, new_synapse->id);

	new_synapse->inputs = g_list_append(new_synapse->inputs,
	new_input);
	}
	}
	}

	return new_synapse;
	}

	/*!
	* \internal
	* \brief Unpack transition graph XML
	*
	* \param[in] xml_graph Transition graph XML to unpack
	* \param[in] reference Where the XML came from (for logging)
	*
	* \return Newly allocated transition graph on success, NULL otherwise
	* \note The caller is responsible for freeing the return value using
	* pcmk__free_graph().
	* \note The XML is expected to be structured like:
	<transition_graph ...>
	<synapse id="0">
	<action_set>
	<rsc_op id="2" ...>
	...
	</action_set>
	<inputs>
	<rsc_op id="1" ...
	...
	</inputs>
	</synapse>
	...
	</transition_graph>
	*/
	pcmk__graph_t *
	pcmk__unpack_graph(const xmlNode xml_graph, const char reference)
	{
	pcmk__graph_t *new_graph = NULL;

	new_graph = calloc(1, sizeof(pcmk__graph_t));
	if (new_graph == NULL) {
	return NULL;
	}

	new_graph->source = strdup(pcmk__s(reference, "unknown"));
	if (new_graph->source == NULL) {
	pcmk__free_graph(new_graph);
	return NULL;
	}

	new_graph->completion_action = pcmk__graph_done;

	// Parse top-level attributes from PCMK__XE_TRANSITION_GRAPH
	if (xml_graph != NULL) {
	const char *buf = crm_element_value(xml_graph, "transition_id");

	CRM_CHECK(buf != NULL,
	pcmk__free_graph(new_graph); return NULL);
	pcmk__scan_min_int(buf, &(new_graph->id), 1);

	buf = crm_element_value(xml_graph, PCMK_OPT_CLUSTER_DELAY);
	CRM_CHECK(buf != NULL,
	pcmk__free_graph(new_graph); return NULL);
	pcmk_parse_interval_spec(buf, &(new_graph->network_delay));

	buf = crm_element_value(xml_graph, PCMK_OPT_STONITH_TIMEOUT);
	if (buf == NULL) {
	new_graph->stonith_timeout = new_graph->network_delay;
	} else {
	pcmk_parse_interval_spec(buf, &(new_graph->stonith_timeout));
	}

	// Use 0 (dynamic limit) as default/invalid, -1 (no limit) as minimum
	buf = crm_element_value(xml_graph, PCMK_OPT_BATCH_LIMIT);
	if ((buf == NULL)
	\|\| (pcmk__scan_min_int(buf, &(new_graph->batch_limit),
	-1) != pcmk_rc_ok)) {
	new_graph->batch_limit = 0;
	}

	buf = crm_element_value(xml_graph, PCMK_OPT_MIGRATION_LIMIT);
	pcmk__scan_min_int(buf, &(new_graph->migration_limit), -1);

	new_graph->failed_stop_offset =
	crm_element_value_copy(xml_graph, "failed-stop-offset");
	new_graph->failed_start_offset =
	crm_element_value_copy(xml_graph, "failed-start-offset");

	if (crm_element_value_epoch(xml_graph, "recheck-by",
	&(new_graph->recheck_by)) != pcmk_ok) {
	new_graph->recheck_by = 0;
	}
	}

	// Unpack each child <synapse> element
	for (const xmlNode *synapse_xml = pcmk__xe_first_child(xml_graph,
	"synapse", NULL,
	NULL);
	synapse_xml != NULL; synapse_xml = pcmk__xe_next_same(synapse_xml)) {

	pcmk__graph_synapse_t *new_synapse = unpack_synapse(new_graph,
	synapse_xml);

	if (new_synapse != NULL) {
	new_graph->synapses = g_list_append(new_graph->synapses,
	new_synapse);
	}
	}

	crm_debug("Unpacked transition %d from %s: %d actions in %d synapses",
	new_graph->id, new_graph->source, new_graph->num_actions,
	new_graph->num_synapses);

	return new_graph;
	}


	/*
	* Other transition graph utilities
	*/

	/*!
	* \internal
	* \brief Synthesize an executor event from a graph action
	*
	* \param[in] resource If not NULL, use greater call ID than in this XML
	* \param[in] action Graph action
	* \param[in] status What to use as event execution status
	* \param[in] rc What to use as event exit status
	* \param[in] exit_reason What to use as event exit reason
	*
	* \return Newly allocated executor event on success, or NULL otherwise
	*/
	lrmd_event_data_t *
	pcmk__event_from_graph_action(const xmlNode *resource,
	const pcmk__graph_action_t *action,
	int status, int rc, const char *exit_reason)
	{
	lrmd_event_data_t *op = NULL;
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;
	xmlNode *action_resource = NULL;

	CRM_CHECK(action != NULL, return NULL);
	CRM_CHECK(action->type == pcmk__rsc_graph_action, return NULL);

	action_resource = pcmk__xe_first_child(action->xml, PCMK_XE_PRIMITIVE, NULL,
	NULL);
	CRM_CHECK(action_resource != NULL, crm_log_xml_warn(action->xml, "invalid");
	return NULL);

	op = lrmd_new_event(pcmk__xe_id(action_resource),
	crm_element_value(action->xml, PCMK_XA_OPERATION),
	action->interval_ms);
	lrmd__set_result(op, rc, status, exit_reason);
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;
	op->params = pcmk__strkey_table(free, free);

	g_hash_table_iter_init(&iter, action->params);
	while (g_hash_table_iter_next(&iter, (void )&name, (void )&value)) {
	pcmk__insert_dup(op->params, name, value);
	}

	for (xmlNode *xop = pcmk__xe_first_child(resource, NULL, NULL, NULL);
	xop != NULL; xop = pcmk__xe_next(xop)) {

	int tmp = 0;

	crm_element_value_int(xop, PCMK__XA_CALL_ID, &tmp);
	crm_debug("Got call_id=%d for %s", tmp, pcmk__xe_id(resource));
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}

	op->call_id++;
	return op;
	}
	diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
	index 5fb7687f2e..77bf76edfa 100644
	--- a/lib/pacemaker/pcmk_sched_ordering.c
	+++ b/lib/pacemaker/pcmk_sched_ordering.c
	@@ -1,1510 +1,1487 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h> // PRIx32
	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	enum pe_order_kind {
	pe_order_kind_optional,
	pe_order_kind_mandatory,
	pe_order_kind_serialize,
	};

	enum ordering_symmetry {
	ordering_asymmetric, // the only relation in an asymmetric ordering
	ordering_symmetric, // the normal relation in a symmetric ordering
	ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
	};

	#define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do { \
	__rsc = pcmk__find_constraint_resource(scheduler->priv->resources, \
	__name); \
	if (__rsc == NULL) { \
	pcmk__config_err("%s: No resource found for %s", __set, __name);\
	return pcmk_rc_unpack_error; \
	} \
	} while (0)

	static const char *
	invert_action(const char *action)
	{
	if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
	return PCMK_ACTION_STOP;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return PCMK_ACTION_START;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTE;

	} else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
	return PCMK_ACTION_DEMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return PCMK_ACTION_PROMOTED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return PCMK_ACTION_STOPPED;

	} else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return PCMK_ACTION_RUNNING;
	}
	pcmk__config_warn("Unknown action '%s' specified in order constraint",
	action);
	return NULL;
	}

	static enum pe_order_kind
	get_ordering_type(const xmlNode *xml_obj)
	{
	enum pe_order_kind kind_e = pe_order_kind_mandatory;
	const char *kind = crm_element_value(xml_obj, PCMK_XA_KIND);

	if (kind == NULL) {
	const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);

	kind_e = pe_order_kind_mandatory;

	if (score) {
	// @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
	int score_i = 0;

	(void) pcmk_parse_score(score, &score_i, 0);
	if (score_i == 0) {
	kind_e = pe_order_kind_optional;
	}
	pcmk__warn_once(pcmk__wo_order_score,
	"Support for '" PCMK_XA_SCORE "' in "
	PCMK_XE_RSC_ORDER " is deprecated and will be "
	"removed in a future release "
	"(use '" PCMK_XA_KIND "' instead)");
	}

	} else if (pcmk__str_eq(kind, PCMK_VALUE_MANDATORY, pcmk__str_none)) {
	kind_e = pe_order_kind_mandatory;

	} else if (pcmk__str_eq(kind, PCMK_VALUE_OPTIONAL, pcmk__str_none)) {
	kind_e = pe_order_kind_optional;

	} else if (pcmk__str_eq(kind, PCMK_VALUE_SERIALIZE, pcmk__str_none)) {
	kind_e = pe_order_kind_serialize;

	} else {
	pcmk__config_err("Resetting '" PCMK_XA_KIND "' for constraint %s to "
	"'" PCMK_VALUE_MANDATORY "' because '%s' is not valid",
	pcmk__s(pcmk__xe_id(xml_obj), "missing ID"), kind);
	}
	return kind_e;
	}

	/*!
	* \internal
	* \brief Get ordering symmetry from XML
	*
	* \param[in] xml_obj Ordering XML
	* \param[in] parent_kind Default ordering kind
	* \param[in] parent_symmetrical_s Parent element's \c PCMK_XA_SYMMETRICAL
	* setting, if any
	*
	* \retval ordering_symmetric Ordering is symmetric
	* \retval ordering_asymmetric Ordering is asymmetric
	*/
	static enum ordering_symmetry
	get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
	const char *parent_symmetrical_s)
	{
	int rc = pcmk_rc_ok;
	bool symmetric = false;
	enum pe_order_kind kind = parent_kind; // Default to parent's kind

	// Check ordering XML for explicit kind
	if ((crm_element_value(xml_obj, PCMK_XA_KIND) != NULL)
	\|\| (crm_element_value(xml_obj, PCMK_XA_SCORE) != NULL)) {
	kind = get_ordering_type(xml_obj);
	}

	// Check ordering XML (and parent) for explicit PCMK_XA_SYMMETRICAL setting
	rc = pcmk__xe_get_bool_attr(xml_obj, PCMK_XA_SYMMETRICAL, &symmetric);

	if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
	symmetric = crm_is_true(parent_symmetrical_s);
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	if (symmetric) {
	if (kind == pe_order_kind_serialize) {
	pcmk__config_warn("Ignoring " PCMK_XA_SYMMETRICAL
	" for '%s' because not valid with "
	PCMK_XA_KIND " of '" PCMK_VALUE_SERIALIZE "'",
	pcmk__xe_id(xml_obj));
	} else {
	return ordering_symmetric;
	}
	}
	return ordering_asymmetric;
	}

	// Use default symmetry
	if (kind == pe_order_kind_serialize) {
	return ordering_asymmetric;
	}
	return ordering_symmetric;
	}

	/*!
	* \internal
	* \brief Get ordering flags appropriate to ordering kind
	*
	* \param[in] kind Ordering kind
	* \param[in] first Action name for 'first' action
	* \param[in] symmetry This ordering's symmetry role
	*
	* \return Minimal ordering flags appropriate to \p kind
	*/
	static uint32_t
	ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
	enum ordering_symmetry symmetry)
	{
	uint32_t flags = pcmk__ar_none; // so we trace-log all flags set

	switch (kind) {
	case pe_order_kind_optional:
	pcmk__set_relation_flags(flags, pcmk__ar_ordered);
	break;

	case pe_order_kind_serialize:
	/* This flag is not used anywhere directly but means the relation
	* will not match an equality comparison against pcmk__ar_none or
	* pcmk__ar_ordered.
	*/
	pcmk__set_relation_flags(flags, pcmk__ar_serialize);
	break;

	case pe_order_kind_mandatory:
	pcmk__set_relation_flags(flags, pcmk__ar_ordered);
	switch (symmetry) {
	case ordering_asymmetric:
	pcmk__set_relation_flags(flags, pcmk__ar_asymmetric);
	break;

	case ordering_symmetric:
	pcmk__set_relation_flags(flags,
	pcmk__ar_first_implies_then);
	if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	pcmk__set_relation_flags(flags,
	pcmk__ar_unrunnable_first_blocks);
	}
	break;

	case ordering_symmetric_inverse:
	pcmk__set_relation_flags(flags,
	pcmk__ar_then_implies_first);
	break;
	}
	break;
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Find resource corresponding to ID specified in ordering
	*
	* \param[in] xml Ordering XML
	* \param[in] resource_attr XML attribute name for resource ID
	* \param[in] scheduler Scheduler data
	*
	* \return Resource corresponding to \p id, or NULL if none
	*/
	static pcmk_resource_t *
	get_ordering_resource(const xmlNode xml, const char resource_attr,
	const pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(xml, resource_attr);

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring constraint '%s' without %s",
	pcmk__xe_id(xml), resource_attr);
	return NULL;
	}

	rsc = pcmk__find_constraint_resource(scheduler->priv->resources, rsc_id);
	if (rsc == NULL) {
	pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
	"does not exist", pcmk__xe_id(xml), rsc_id);
	return NULL;
	}

	return rsc;
	}

	/*!
	* \internal
	* \brief Determine minimum number of 'first' instances required in ordering
	*
	* \param[in] rsc 'First' resource in ordering
	* \param[in] xml Ordering XML
	*
	* \return Minimum 'first' instances required (or 0 if not applicable)
	*/
	static int
	get_minimum_first_instances(const pcmk_resource_t rsc, const xmlNode xml)
	{
	const char *clone_min = NULL;
	bool require_all = false;

	if (!pcmk__is_clone(rsc)) {
	return 0;
	}

	clone_min = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CLONE_MIN);
	if (clone_min != NULL) {
	int clone_min_int = 0;

	pcmk__scan_min_int(clone_min, &clone_min_int, 0);
	return clone_min_int;
	}

	/* @COMPAT 1.1.13:
	* PCMK_XA_REQUIRE_ALL=PCMK_VALUE_FALSE is deprecated equivalent of
	* PCMK_META_CLONE_MIN=1
	*/
	if (pcmk__xe_get_bool_attr(xml, PCMK_XA_REQUIRE_ALL,
	&require_all) != ENODATA) {
	pcmk__warn_once(pcmk__wo_require_all,
	"Support for " PCMK_XA_REQUIRE_ALL " in ordering "
	"constraints is deprecated and will be removed in a "
	"future release (use " PCMK_META_CLONE_MIN " clone "
	"meta-attribute instead)");
	if (!require_all) {
	return 1;
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Create orderings for a constraint with \c PCMK_META_CLONE_MIN > 0
	*
	* \param[in] id Ordering ID
	* \param[in,out] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	* \param[in] flags Ordering flags
	* \param[in] clone_min Minimum required instances of 'first'
	*/
	static void
	clone_min_ordering(const char *id,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then,
	uint32_t flags, int clone_min)
	{
	// Create a pseudo-action for when the minimum instances are active
	char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
	pcmk_action_t *clone_min_met = get_pseudo_op(task,
	rsc_first->priv->scheduler);

	free(task);

	/* Require the pseudo-action to have the required number of actions to be
	* considered runnable before allowing the pseudo-action to be runnable.
	*/
	clone_min_met->required_runnable_before = clone_min;

	// Order the actions for each clone instance before the pseudo-action
	for (GList *iter = rsc_first->priv->children;
	iter != NULL; iter = iter->next) {

	pcmk_resource_t *child = iter->data;

	pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
	NULL, NULL, NULL, clone_min_met,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	rsc_first->priv->scheduler);
	}

	// Order "then" action after the pseudo-action (if runnable)
	pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
	pcmk__op_key(rsc_then->id, action_then, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	rsc_first->priv->scheduler);
	}

	-/*!
	- * \internal
	- * \brief Update ordering flags for restart-type=restart
	- *
	- * \param[in] rsc 'Then' resource in ordering
	- * \param[in] kind Ordering kind
	- * \param[in] flag Ordering flag to set (when applicable)
	- * \param[in,out] flags Ordering flag set to update
	- *
	- * \compat The \c PCMK__META_RESTART_TYPE resource meta-attribute is deprecated.
	- * Eventually, it will be removed, and \c pcmk__restart_ignore will be
	- * the only behavior, at which time this can just be removed entirely.
	- */
	-#define handle_restart_type(rsc, kind, flag, flags) do { \
	- if (((kind) == pe_order_kind_optional) \
	- && ((rsc)->priv->restart_type == pcmk__restart_restart)) { \
	- pcmk__set_relation_flags((flags), (flag)); \
	- } \
	- } while (0)
	-
	/*!
	* \internal
	* \brief Create new ordering for inverse of symmetric constraint
	*
	* \param[in] id Ordering ID (for logging only)
	* \param[in] kind Ordering kind
	* \param[in] rsc_first 'First' resource in ordering (a clone)
	* \param[in] action_first 'First' action in ordering
	* \param[in,out] rsc_then 'Then' resource in ordering
	* \param[in] action_then 'Then' action in ordering
	*/
	static void
	inverse_ordering(const char *id, enum pe_order_kind kind,
	pcmk_resource_t rsc_first, const char action_first,
	pcmk_resource_t rsc_then, const char action_then)
	{
	action_then = invert_action(action_then);
	action_first = invert_action(action_first);
	if ((action_then == NULL) \|\| (action_first == NULL)) {
	pcmk__config_warn("Cannot invert constraint '%s' "
	"(please specify inverse manually)", id);
	} else {
	uint32_t flags = ordering_flags_for_kind(kind, action_first,
	ordering_symmetric_inverse);

	- handle_restart_type(rsc_then, kind, pcmk__ar_then_implies_first, flags);
	pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
	action_first, flags);
	}
	}

	static void
	unpack_simple_rsc_order(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *rsc_then = NULL;
	pcmk_resource_t *rsc_first = NULL;
	int min_required_before = 0;
	enum pe_order_kind kind = pe_order_kind_mandatory;
	uint32_t flags = pcmk__ar_none;
	enum ordering_symmetry symmetry;

	const char *action_then = NULL;
	const char *action_first = NULL;
	const char *id = NULL;

	CRM_CHECK(xml_obj != NULL, return);

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return;
	}

	rsc_first = get_ordering_resource(xml_obj, PCMK_XA_FIRST, scheduler);
	if (rsc_first == NULL) {
	return;
	}

	rsc_then = get_ordering_resource(xml_obj, PCMK_XA_THEN, scheduler);
	if (rsc_then == NULL) {
	return;
	}

	action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
	if (action_first == NULL) {
	action_first = PCMK_ACTION_START;
	}

	action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);
	if (action_then == NULL) {
	action_then = action_first;
	}

	kind = get_ordering_type(xml_obj);

	symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
	flags = ordering_flags_for_kind(kind, action_first, symmetry);

	- handle_restart_type(rsc_then, kind, pcmk__ar_first_implies_then, flags);
	-
	/* If there is a minimum number of instances that must be runnable before
	* the 'then' action is runnable, we use a pseudo-action for convenience:
	* minimum number of clone instances have runnable actions ->
	* pseudo-action is runnable -> dependency is runnable.
	*/
	min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
	if (min_required_before > 0) {
	clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
	flags, min_required_before);
	} else {
	pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
	action_then, flags);
	}

	if (symmetry == ordering_symmetric) {
	inverse_ordering(id, kind, rsc_first, action_first,
	rsc_then, action_then);
	}
	}

	/*!
	* \internal
	* \brief Create a new ordering between two actions
	*
	* \param[in,out] first_rsc Resource for 'first' action (if NULL and
	* \p first_action is a resource action, that
	* resource will be used)
	* \param[in,out] first_action_task Action key for 'first' action (if NULL and
	* \p first_action is not NULL, its UUID will
	* be used)
	* \param[in,out] first_action 'first' action (if NULL, \p first_rsc and
	* \p first_action_task must be set)
	*
	* \param[in] then_rsc Resource for 'then' action (if NULL and
	* \p then_action is a resource action, that
	* resource will be used)
	* \param[in,out] then_action_task Action key for 'then' action (if NULL and
	* \p then_action is not NULL, its UUID will
	* be used)
	* \param[in] then_action 'then' action (if NULL, \p then_rsc and
	* \p then_action_task must be set)
	*
	* \param[in] flags Group of enum pcmk__action_relation_flags
	* \param[in,out] sched Scheduler data to add ordering to
	*
	* \note This function takes ownership of first_action_task and
	* then_action_task, which do not need to be freed by the caller.
	*/
	void
	pcmk__new_ordering(pcmk_resource_t first_rsc, char first_action_task,
	pcmk_action_t first_action, pcmk_resource_t then_rsc,
	char then_action_task, pcmk_action_t then_action,
	uint32_t flags, pcmk_scheduler_t *sched)
	{
	pcmk__action_relation_t *order = NULL;

	// One of action or resource must be specified for each side
	CRM_CHECK(((first_action != NULL) \|\| (first_rsc != NULL))
	&& ((then_action != NULL) \|\| (then_rsc != NULL)),
	free(first_action_task); free(then_action_task); return);

	if ((first_rsc == NULL) && (first_action != NULL)) {
	first_rsc = first_action->rsc;
	}
	if ((then_rsc == NULL) && (then_action != NULL)) {
	then_rsc = then_action->rsc;
	}

	order = pcmk__assert_alloc(1, sizeof(pcmk__action_relation_t));

	order->id = sched->priv->next_ordering_id++;
	order->flags = flags;
	order->rsc1 = first_rsc;
	order->rsc2 = then_rsc;
	order->action1 = first_action;
	order->action2 = then_action;
	order->task1 = first_action_task;
	order->task2 = then_action_task;

	if ((order->task1 == NULL) && (first_action != NULL)) {
	order->task1 = strdup(first_action->uuid);
	}

	if ((order->task2 == NULL) && (then_action != NULL)) {
	order->task2 = strdup(then_action->uuid);
	}

	if ((order->rsc1 == NULL) && (first_action != NULL)) {
	order->rsc1 = first_action->rsc;
	}

	if ((order->rsc2 == NULL) && (then_action != NULL)) {
	order->rsc2 = then_action->rsc;
	}

	pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s",
	(sched->priv->next_ordering_id - 1),
	pcmk__s(order->task1, "an underspecified action"),
	pcmk__s(order->task2, "an underspecified action"));

	sched->priv->ordering_constraints =
	g_list_prepend(sched->priv->ordering_constraints, order);
	pcmk__order_migration_equivalents(order);
	}

	/*!
	* \brief Unpack a set in an ordering constraint
	*
	* \param[in] set Set XML to unpack
	* \param[in] parent_kind \c PCMK_XE_RSC_ORDER XML \c PCMK_XA_KIND
	* attribute
	* \param[in] parent_symmetrical_s \c PCMK_XE_RSC_ORDER XML
	* \c PCMK_XA_SYMMETRICAL attribute
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
	const char parent_symmetrical_s, pcmk_scheduler_t scheduler)
	{
	GList *set_iter = NULL;
	GList *resources = NULL;

	pcmk_resource_t *last = NULL;
	pcmk_resource_t *resource = NULL;

	int local_kind = parent_kind;
	bool sequential = false;
	uint32_t flags = pcmk__ar_ordered;
	enum ordering_symmetry symmetry;

	char *key = NULL;
	const char *id = pcmk__xe_id(set);
	const char *action = crm_element_value(set, PCMK_XA_ACTION);
	const char *sequential_s = crm_element_value(set, PCMK_XA_SEQUENTIAL);
	const char *kind_s = crm_element_value(set, PCMK_XA_KIND);

	if (action == NULL) {
	action = PCMK_ACTION_START;
	}

	if (kind_s) {
	local_kind = get_ordering_type(set);
	}
	if (sequential_s == NULL) {
	sequential_s = "1";
	}

	sequential = crm_is_true(sequential_s);

	symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
	flags = ordering_flags_for_kind(local_kind, action, symmetry);

	for (const xmlNode *xml_rsc = pcmk__xe_first_child(set,
	PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, resource, pcmk__xe_id(xml_rsc));
	resources = g_list_append(resources, resource);
	}

	if (pcmk__list_of_1(resources)) {
	crm_trace("Single set: %s", id);
	goto done;
	}

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	key = pcmk__op_key(resource->id, action, 0);

	if (local_kind == pe_order_kind_serialize) {
	/* Serialize before everything that comes after */

	for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
	pcmk_resource_t *then_rsc = iter->data;
	char *then_key = pcmk__op_key(then_rsc->id, action, 0);

	pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
	then_key, NULL, flags, scheduler);
	}

	} else if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(last, action, resource, action,
	flags);
	}
	last = resource;
	}
	free(key);
	}

	if (symmetry == ordering_asymmetric) {
	goto done;
	}

	last = NULL;
	action = invert_action(action);

	flags = ordering_flags_for_kind(local_kind, action,
	ordering_symmetric_inverse);

	set_iter = resources;
	while (set_iter != NULL) {
	resource = (pcmk_resource_t *) set_iter->data;
	set_iter = set_iter->next;

	if (sequential) {
	if (last != NULL) {
	pcmk__order_resource_actions(resource, action, last, action,
	flags);
	}
	last = resource;
	}
	}

	done:
	g_list_free(resources);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Order two resource sets relative to each other
	*
	* \param[in] id Ordering ID (for logging)
	* \param[in] set1 First listed set
	* \param[in] set2 Second listed set
	* \param[in] kind Ordering kind
	* \param[in,out] scheduler Scheduler data
	* \param[in] symmetry Which ordering symmetry applies to this relation
	*
	* \return Standard Pacemaker return code
	*/
	static int
	order_rsc_sets(const char id, const xmlNode set1, const xmlNode *set2,
	enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
	enum ordering_symmetry symmetry)
	{

	const xmlNode *xml_rsc = NULL;
	const xmlNode *xml_rsc_2 = NULL;

	pcmk_resource_t *rsc_1 = NULL;
	pcmk_resource_t *rsc_2 = NULL;

	const char *action_1 = crm_element_value(set1, PCMK_XA_ACTION);
	const char *action_2 = crm_element_value(set2, PCMK_XA_ACTION);

	uint32_t flags = pcmk__ar_none;

	bool require_all = true;

	(void) pcmk__xe_get_bool_attr(set1, PCMK_XA_REQUIRE_ALL, &require_all);

	if (action_1 == NULL) {
	action_1 = PCMK_ACTION_START;
	}

	if (action_2 == NULL) {
	action_2 = PCMK_ACTION_START;
	}

	if (symmetry == ordering_symmetric_inverse) {
	action_1 = invert_action(action_1);
	action_2 = invert_action(action_2);
	}

	if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
	\|\| pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
	/* Assuming: A -> ( B \|\| C) -> D
	* The one-or-more logic only applies during the start/promote phase.
	* During shutdown neither B nor can shutdown until D is down, so simply
	* turn require_all back on.
	*/
	require_all = true;
	}

	flags = ordering_flags_for_kind(kind, action_1, symmetry);

	/* If we have an unordered set1, whether it is sequential or not is
	* irrelevant in regards to set2.
	*/
	if (!require_all) {
	char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s",
	pcmk__xe_id(set1));
	pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);

	free(task);
	unordered_action->required_runnable_before = 1;

	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));

	/* Add an ordering constraint between every element in set1 and the
	* pseudo action. If any action in set1 is runnable the pseudo
	* action will be runnable.
	*/
	pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
	NULL, NULL, NULL, unordered_action,
	pcmk__ar_min_runnable
	\|pcmk__ar_first_implies_then_graphed,
	scheduler);
	}
	for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next_same(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));

	/* Add an ordering constraint between the pseudo-action and every
	* element in set2. If the pseudo-action is runnable, every action
	* in set2 will be runnable.
	*/
	pcmk__new_ordering(NULL, NULL, unordered_action,
	rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
	NULL, flags\|pcmk__ar_unrunnable_first_blocks,
	scheduler);
	}

	return pcmk_rc_ok;
	}

	if (pcmk__xe_attr_is_true(set1, PCMK_XA_SEQUENTIAL)) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the first one
	xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
	}

	} else {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	rid = pcmk__xe_id(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
	}
	}

	if (pcmk__xe_attr_is_true(set2, PCMK_XA_SEQUENTIAL)) {
	if (symmetry == ordering_symmetric_inverse) {
	// Get the last one
	const char *rid = NULL;

	for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	rid = pcmk__xe_id(xml_rsc);
	}
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);

	} else {
	// Get the first one
	xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	if (xml_rsc != NULL) {
	EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
	}
	}
	}

	if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);

	} else if (rsc_1 != NULL) {
	for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else if (rsc_2 != NULL) {
	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
	flags);
	}

	} else {
	for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
	NULL);
	xml_rsc != NULL; xml_rsc = pcmk__xe_next_same(xml_rsc)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));

	for (xmlNode *xml_rsc_2 = pcmk__xe_first_child(set2,
	PCMK_XE_RESOURCE_REF,
	NULL, NULL);
	xml_rsc_2 != NULL; xml_rsc_2 = pcmk__xe_next_same(xml_rsc_2)) {

	EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));
	pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
	action_2, flags);
	}
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief If an ordering constraint uses resource tags, expand them
	*
	* \param[in,out] xml_obj Ordering constraint XML
	* \param[out] expanded_xml Equivalent XML with tags expanded
	* \param[in] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
	* and pcmk_rc_unpack_error on invalid configuration)
	*/
	static int
	unpack_order_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	const pcmk_scheduler_t *scheduler)
	{
	const char *id_first = NULL;
	const char *id_then = NULL;
	const char *action_first = NULL;
	const char *action_then = NULL;

	pcmk_resource_t *rsc_first = NULL;
	pcmk_resource_t *rsc_then = NULL;
	pcmk__idref_t *tag_first = NULL;
	pcmk__idref_t *tag_then = NULL;

	xmlNode *rsc_set_first = NULL;
	xmlNode *rsc_set_then = NULL;
	bool any_sets = false;

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
	return pcmk_rc_ok;
	}

	id_first = crm_element_value(xml_obj, PCMK_XA_FIRST);
	id_then = crm_element_value(xml_obj, PCMK_XA_THEN);
	if ((id_first == NULL) \|\| (id_then == NULL)) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
	&tag_first)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag",
	pcmk__xe_id(xml_obj), id_first);
	return pcmk_rc_unpack_error;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
	&tag_then)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag",
	pcmk__xe_id(xml_obj), id_then);
	return pcmk_rc_unpack_error;
	}

	if ((rsc_first != NULL) && (rsc_then != NULL)) {
	// Neither side references a template or tag
	return pcmk_rc_ok;
	}

	action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
	action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);

	*expanded_xml = pcmk__xml_copy(NULL, xml_obj);

	/* Convert template/tag reference in PCMK_XA_FIRST into constraint
	* PCMK_XE_RESOURCE_SET
	*/
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, PCMK_XA_FIRST, true,
	scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_first != NULL) {
	if (action_first != NULL) {
	/* Move PCMK_XA_FIRST_ACTION into converted PCMK_XE_RESOURCE_SET as
	* PCMK_XA_ACTION
	*/
	crm_xml_add(rsc_set_first, PCMK_XA_ACTION, action_first);
	pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_FIRST_ACTION);
	}
	any_sets = true;
	}

	/* Convert template/tag reference in PCMK_XA_THEN into constraint
	* PCMK_XE_RESOURCE_SET
	*/
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, PCMK_XA_THEN, true,
	scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set_then != NULL) {
	if (action_then != NULL) {
	/* Move PCMK_XA_THEN_ACTION into converted PCMK_XE_RESOURCE_SET as
	* PCMK_XA_ACTION
	*/
	crm_xml_add(rsc_set_then, PCMK_XA_ACTION, action_then);
	pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_THEN_ACTION);
	}
	any_sets = true;
	}

	if (any_sets) {
	crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
	} else {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Unpack ordering constraint XML
	*
	* \param[in,out] xml_obj Ordering constraint XML to unpack
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_ordering(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	xmlNode *last = NULL;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char *invert = crm_element_value(xml_obj, PCMK_XA_SYMMETRICAL);
	enum pe_order_kind kind = get_ordering_type(xml_obj);

	enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
	NULL);

	// Expand any resource tags in the constraint XML
	if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}
	if (expanded_xml != NULL) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	// If the constraint has resource sets, unpack them
	for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
	set != NULL; set = pcmk__xe_next_same(set)) {

	set = pcmk__xe_resolve_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml != NULL) {
	pcmk__xml_free(expanded_xml);
	}
	return;
	}

	if (last != NULL) {

	if (order_rsc_sets(id, last, set, kind, scheduler,
	symmetry) != pcmk_rc_ok) {
	if (expanded_xml != NULL) {
	pcmk__xml_free(expanded_xml);
	}
	return;
	}

	if ((symmetry == ordering_symmetric)
	&& (order_rsc_sets(id, set, last, kind, scheduler,
	ordering_symmetric_inverse) != pcmk_rc_ok)) {
	if (expanded_xml != NULL) {
	pcmk__xml_free(expanded_xml);
	}
	return;
	}

	}
	last = set;
	}

	if (expanded_xml) {
	pcmk__xml_free(expanded_xml);
	xml_obj = orig_xml;
	}

	// If the constraint has no resource sets, unpack it as a simple ordering
	if (last == NULL) {
	return unpack_simple_rsc_order(xml_obj, scheduler);
	}
	}

	static bool
	ordering_is_invalid(pcmk_action_t action, pcmk__related_action_t input)
	{
	/* Prevent user-defined ordering constraints between resources
	* running in a guest node and the resource that defines that node.
	*/
	if (!pcmk_is_set(input->flags, pcmk__ar_guest_allowed)
	&& (input->action->rsc != NULL)
	&& pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {

	pcmk__config_warn("Invalid ordering constraint between %s and %s",
	input->action->rsc->id, action->rsc->id);
	return true;
	}

	/* If there's an order like
	* "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
	*
	* then rscA is being migrated from node1 to node2, while rscB is being
	* migrated from node2 to node1. If there would be a graph loop,
	* break the order "load_stopped_node2" -> "rscA_migrate_to node1".
	*/
	if ((input->flags == pcmk__ar_if_on_same_node_or_target)
	&& (action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
	&& pcmk__graph_has_loop(action, action, input)) {
	return true;
	}

	return false;
	}

	void
	pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->priv->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t action = (pcmk_action_t ) iter->data;
	pcmk__related_action_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = input_iter->data;
	if (ordering_is_invalid(action, input)) {
	input->flags = pcmk__ar_none;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Order stops on a node before the node's shutdown
	*
	* \param[in,out] node Node being shut down
	* \param[in] shutdown_op Shutdown action for node
	*/
	void
	pcmk__order_stops_before_shutdown(pcmk_node_t node, pcmk_action_t shutdown_op)
	{
	for (GList *iter = node->priv->scheduler->priv->actions;
	iter != NULL; iter = iter->next) {

	pcmk_action_t action = (pcmk_action_t ) iter->data;

	// Only stops on the node shutting down are relevant
	if (!pcmk__same_node(action->node, node)
	\|\| !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	continue;
	}

	// Resources and nodes in maintenance mode won't be touched

	if (pcmk_is_set(action->rsc->flags, pcmk__rsc_maintenance)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource in maintenance mode",
	action->uuid, pcmk__node_name(node));
	continue;

	} else if (node->details->maintenance) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"node in maintenance mode",
	action->uuid, pcmk__node_name(node));
	continue;
	}

	/* Don't touch a resource that is unmanaged or blocked, to avoid
	* blocking the shutdown (though if another action depends on this one,
	* we may still end up blocking)
	*/
	if (!pcmk_any_flags_set(action->rsc->flags,
	pcmk__rsc_managed\|pcmk__rsc_blocked)) {
	pcmk__rsc_trace(action->rsc,
	"Not ordering %s before shutdown of %s because "
	"resource is unmanaged or blocked",
	action->uuid, pcmk__node_name(node));
	continue;
	}

	pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
	action->uuid, pcmk__node_name(node));
	pcmk__clear_action_flags(action, pcmk__action_optional);
	pcmk__new_ordering(action->rsc, NULL, action, NULL,
	strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
	pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks,
	node->priv->scheduler);
	}
	}

	/*!
	* \brief Find resource actions matching directly or as child
	*
	* \param[in] rsc Resource to check
	* \param[in] original_key Action key to search for (possibly referencing
	* parent of \rsc)
	*
	* \return Newly allocated list of matching actions
	* \note It is the caller's responsibility to free the result with g_list_free()
	*/
	static GList *
	find_actions_by_task(const pcmk_resource_t rsc, const char original_key)
	{
	// Search under given task key directly
	GList *list = find_actions(rsc->priv->actions, original_key, NULL);

	if (list == NULL) {
	// Search again using this resource's ID
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms),
	return NULL);
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(rsc->priv->actions, key, NULL);
	free(key);
	free(task);
	}
	return list;
	}

	/*!
	* \internal
	* \brief Order relevant resource actions after a given action
	*
	* \param[in,out] first_action Action to order after (or NULL if none runnable)
	* \param[in] rsc Resource whose actions should be ordered
	* \param[in,out] order Ordering constraint being applied
	*/
	static void
	order_resource_actions_after(pcmk_action_t *first_action,
	const pcmk_resource_t *rsc,
	pcmk__action_relation_t *order)
	{
	GList *then_actions = NULL;
	uint32_t flags = pcmk__ar_none;

	CRM_CHECK((rsc != NULL) && (order != NULL), return);

	flags = order->flags;
	pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
	order->id, rsc->id);

	if (order->action2 != NULL) {
	then_actions = g_list_prepend(NULL, order->action2);

	} else {
	then_actions = find_actions_by_task(rsc, order->task2);
	}

	if (then_actions == NULL) {
	pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
	order->id, order->task2, rsc->id);
	return;
	}

	if ((first_action != NULL) && (first_action->rsc == rsc)
	&& pcmk_is_set(first_action->flags, pcmk__action_migration_abort)) {

	pcmk__rsc_trace(rsc,
	"Detected dangling migration ordering (%s then %s %s)",
	first_action->uuid, order->task2, rsc->id);
	pcmk__clear_relation_flags(flags, pcmk__ar_first_implies_then);
	}

	if ((first_action == NULL)
	&& !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {

	pcmk__rsc_debug(rsc,
	"Ignoring ordering %d for %s: No first action found",
	order->id, rsc->id);
	g_list_free(then_actions);
	return;
	}

	for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then_action_iter = (pcmk_action_t ) iter->data;

	if (first_action != NULL) {
	order_actions(first_action, then_action_iter, flags);
	} else {
	pcmk__clear_action_flags(then_action_iter, pcmk__action_runnable);
	crm_warn("%s of %s is unrunnable because there is no %s of %s "
	"to order it after", then_action_iter->task, rsc->id,
	order->task1, order->rsc1->id);
	}
	}

	g_list_free(then_actions);
	}

	static void
	rsc_order_first(pcmk_resource_t first_rsc, pcmk__action_relation_t order)
	{
	GList *first_actions = NULL;
	pcmk_action_t *first_action = order->action1;
	pcmk_resource_t *then_rsc = order->rsc2;

	pcmk__assert(first_rsc != NULL);
	pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
	order->id, first_rsc->id);

	if (first_action != NULL) {
	first_actions = g_list_prepend(NULL, first_action);

	} else {
	first_actions = find_actions_by_task(first_rsc, order->task1);
	}

	if ((first_actions == NULL) && (first_rsc == then_rsc)) {
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) not found",
	order->id, order->task1, first_rsc->id);

	} else if (first_actions == NULL) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;
	enum rsc_role_e first_role;

	parse_op_key(order->task1, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(first_rsc->id, op_type, interval_ms);

	first_role = first_rsc->priv->fns->state(first_rsc, TRUE);
	if ((first_role == pcmk_role_stopped)
	&& pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else if ((first_role == pcmk_role_unpromoted)
	&& pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	free(key);
	pcmk__rsc_trace(first_rsc,
	"Ignoring constraint %d: first (%s for %s) "
	"not found",
	order->id, order->task1, first_rsc->id);

	} else {
	pcmk__rsc_trace(first_rsc,
	"Creating first (%s for %s) for constraint %d ",
	order->task1, first_rsc->id, order->id);
	first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
	first_rsc->priv->scheduler);
	first_actions = g_list_prepend(NULL, first_action);
	}

	free(op_type);
	}

	if (then_rsc == NULL) {
	if (order->action2 == NULL) {
	pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
	order->id);
	return;
	}
	then_rsc = order->action2->rsc;
	}
	for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
	first_action = iter->data;

	if (then_rsc == NULL) {
	order_actions(first_action, order->action2, order->flags);

	} else {
	order_resource_actions_after(first_action, then_rsc, order);
	}
	}

	g_list_free(first_actions);
	}

	// GFunc to call pcmk__block_colocation_dependents()
	static void
	block_colocation_dependents(gpointer data, gpointer user_data)
	{
	pcmk__block_colocation_dependents(data);
	}

	// GFunc to call pcmk__update_action_for_orderings()
	static void
	update_action_for_orderings(gpointer data, gpointer user_data)
	{
	pcmk__update_action_for_orderings((pcmk_action_t *) data,
	(pcmk_scheduler_t *) user_data);
	}

	/*!
	* \internal
	* \brief Apply all ordering constraints
	*
	* \param[in,out] sched Scheduler data
	*/
	void
	pcmk__apply_orderings(pcmk_scheduler_t *sched)
	{
	crm_trace("Applying ordering constraints");

	/* Ordering constraints need to be processed in the order they were created.
	* rsc_order_first() and order_resource_actions_after() require the relevant
	* actions to already exist in some cases, but rsc_order_first() will create
	* the 'first' action in certain cases. Thus calling rsc_order_first() can
	* change the behavior of later-created orderings.
	*
	* Also, g_list_append() should be avoided for performance reasons, so we
	* prepend orderings when creating them and reverse the list here.
	*
	* @TODO This is brittle and should be carefully redesigned so that the
	* order of creation doesn't matter, and the reverse becomes unneeded.
	*/
	sched->priv->ordering_constraints =
	g_list_reverse(sched->priv->ordering_constraints);

	for (GList *iter = sched->priv->ordering_constraints;
	iter != NULL; iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	pcmk_resource_t *rsc = order->rsc1;

	if (rsc != NULL) {
	rsc_order_first(rsc, order);
	continue;
	}

	rsc = order->rsc2;
	if (rsc != NULL) {
	order_resource_actions_after(order->action1, rsc, order);

	} else {
	crm_trace("Applying ordering constraint %d (non-resource actions)",
	order->id);
	order_actions(order->action1, order->action2, order->flags);
	}
	}

	g_list_foreach(sched->priv->actions, block_colocation_dependents, NULL);

	crm_trace("Ordering probes");
	pcmk__order_probes(sched);

	crm_trace("Updating %d actions", g_list_length(sched->priv->actions));
	g_list_foreach(sched->priv->actions, update_action_for_orderings, sched);

	pcmk__disable_invalid_orderings(sched);
	}

	/*!
	* \internal
	* \brief Order a given action after each action in a given list
	*
	* \param[in,out] after "After" action
	* \param[in,out] list List of "before" actions
	*/
	void
	pcmk__order_after_each(pcmk_action_t after, GList list)
	{
	const char *after_desc = (after->task == NULL)? after->uuid : after->task;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk_action_t before = (pcmk_action_t ) iter->data;
	const char *before_desc = before->task? before->task : before->uuid;

	crm_debug("Ordering %s on %s before %s on %s",
	before_desc, pcmk__node_name(before->node),
	after_desc, pcmk__node_name(after->node));
	order_actions(before, after, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Order promotions and demotions for restarts of a clone or bundle
	*
	* \param[in,out] rsc Clone or bundle to order
	*/
	void
	pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
	{
	// Order start and promote after all instances are stopped
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order stop, start, and promote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order promote after all instances are started
	pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
	rsc, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	// Order demote after all instances are demoted
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
	rsc, PCMK_ACTION_DEMOTED,
	pcmk__ar_ordered);
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 148771ff08..3069a95784 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1291 +1,1274 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pcmk_node_t active_node(const pcmk_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	static pcmk__rsc_methods_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pcmk__rsc_variant
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, PCMK_XE_PRIMITIVE, pcmk__str_casei)) {
	return pcmk__rsc_variant_primitive;

	} else if (pcmk__str_eq(name, PCMK_XE_GROUP, pcmk__str_casei)) {
	return pcmk__rsc_variant_group;

	} else if (pcmk__str_eq(name, PCMK_XE_CLONE, pcmk__str_casei)) {
	return pcmk__rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_BUNDLE, pcmk__str_casei)) {
	return pcmk__rsc_variant_bundle;
	}

	return pcmk__rsc_variant_unknown;
	}

	/*!
	* \internal
	* \brief Insert a meta-attribute if not already present
	*
	* \param[in] key Meta-attribute name
	* \param[in] value Meta-attribute value to add if not already present
	* \param[in,out] table Meta-attribute hash table to insert into
	*
	* \note This is like pcmk__insert_meta() except it won't overwrite existing
	* values.
	*/
	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	GHashTable *table = user_data;

	CRM_CHECK((key != NULL) && (table != NULL), return);
	if (pcmk__str_eq((const char *) value, "#default", pcmk__str_casei)) {
	// @COMPAT Deprecated since 2.1.8
	pcmk__config_warn("Support for setting meta-attributes (such as %s) to "
	"the explicit value '#default' is deprecated and "
	"will be removed in a future release",
	(const char *) key);
	} else if ((value != NULL) && (g_hash_table_lookup(table, key) == NULL)) {
	pcmk__insert_dup(table, (const char ) key, (const char ) value);
	}
	}

	static void
	expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
	pe_rule_eval_data_t *rule_data,
	GHashTable meta_hash, pcmk_scheduler_t scheduler)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pcmk_resource_t *p = rsc->priv->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of
	* PCMK_XE_META_ATTRIBUTES set only in the original xml, and stack it in the
	* hash table. The fixed value of the lower parent resource takes precedence
	* and is not overwritten.
	*/
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->priv->xml, PCMK_XE_META_ATTRIBUTES,
	rule_data, parent_orig_meta, NULL,
	scheduler);
	p = p->priv->parent;
	}

	if (parent_orig_meta != NULL) {
	// This will not overwrite any values already existing for child
	g_hash_table_foreach(parent_orig_meta, dup_attr, meta_hash);
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}

	/*
	* \brief Get fully evaluated resource meta-attributes
	*
	* \param[in,out] meta_hash Where to store evaluated meta-attributes
	* \param[in] rsc Resource to get meta-attributes for
	* \param[in] node Ignored
	* \param[in,out] scheduler Scheduler data
	*/
	void
	get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
	.provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
	.agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.now = scheduler->priv->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->priv->xml);
	a != NULL; a = a->next) {

	if (a->children != NULL) {
	dup_attr((gpointer) a->name, (gpointer) a->children->content,
	meta_hash);
	}
	}

	pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_META_ATTRIBUTES,
	&rule_data, meta_hash, NULL, scheduler);

	/* Set the PCMK_XE_META_ATTRIBUTES explicitly set in the parent resource to
	* the hash table of the child resource. If it is already explicitly set as
	* a child, it will not be overwritten.
	*/
	if (rsc->priv->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->priv->rsc_defaults,
	PCMK_XE_META_ATTRIBUTES, &rule_data, meta_hash,
	NULL, scheduler);

	/* If there is PCMK_XE_META_ATTRIBUTES that the parent resource has not
	* explicitly set, set a value that is not set from PCMK_XE_RSC_DEFAULTS
	* either. The values already set up to this point will not be overwritten.
	*/
	if (rsc->priv->parent != NULL) {
	g_hash_table_foreach(rsc->priv->parent->priv->meta, dup_attr,
	meta_hash);
	}
	}

	/*!
	* \brief Get final values of a resource's instance attributes
	*
	* \param[in,out] instance_attrs Where to store the instance attributes
	* \param[in] rsc Resource to get instance attributes for
	* \param[in] node If not NULL, evaluate rules for this node
	* \param[in,out] scheduler Scheduler data
	*/
	void
	get_rsc_attributes(GHashTable instance_attrs, const pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK((instance_attrs != NULL) && (rsc != NULL) && (scheduler != NULL),
	return);

	rule_data.now = scheduler->priv->now;
	if (node != NULL) {
	rule_data.node_hash = node->priv->attrs;
	}

	// Evaluate resource's own values, then its ancestors' values
	pe__unpack_dataset_nvpairs(rsc->priv->xml, PCMK_XE_INSTANCE_ATTRIBUTES,
	&rule_data, instance_attrs, NULL, scheduler);
	if (rsc->priv->parent != NULL) {
	get_rsc_attributes(instance_attrs, rsc->priv->parent, node, scheduler);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, PCMK_XA_NAME);
	const char *role = crm_element_value(op, PCMK_XA_ROLE);
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK_ROLE_STARTED, PCMK_ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, PCMK_XA_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = pcmk__xe_id(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//" PCMK_XE_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (cib_resources == NULL) {
	pcmk__config_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_first_child(cib_resources, PCMK_XE_TEMPLATE,
	PCMK_XA_ID, template_ref);
	if (template == NULL) {
	pcmk__config_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = pcmk__xml_copy(NULL, template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, PCMK_XA_ID, id);
	crm_xml_add(new_xml, PCMK__META_CLONE,
	crm_element_value(xml_obj, PCMK__META_CLONE));

	template_ops = pcmk__xe_first_child(new_xml, PCMK_XE_OPERATIONS, NULL,
	NULL);

	for (child_xml = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
	child_xml != NULL; child_xml = pcmk__xe_next(child_xml)) {

	xmlNode *new_child = pcmk__xml_copy(new_xml, child_xml);

	if (pcmk__xe_is(new_child, PCMK_XE_OPERATIONS)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops, NULL, NULL, NULL); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops, NULL, NULL, NULL);
	op != NULL; op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	pcmk__xml_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	pcmk__xml_free(template_ops);
	}

	/pcmk__xml_free(expanded_xml); */
	*expanded_xml = new_xml;

	#if 0 /* Disable multi-level templates for now */
	if (!unpack_template(new_xml, expanded_xml, scheduler)) {
	pcmk__xml_free(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}
	#endif

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for processing resource list "
	"of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, PCMK_XA_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = pcmk__xe_id(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	pcmk__add_idref(scheduler->priv->templates, template_ref, id);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether a clone or instance being unpacked is globally unique
	*
	* \param[in] rsc Clone or clone instance to check
	*
	* \return \c true if \p rsc is globally unique according to its
	* meta-attributes, otherwise \c false
	*/
	static bool
	detect_unique(const pcmk_resource_t *rsc)
	{
	const char *value = g_hash_table_lookup(rsc->priv->meta,
	PCMK_META_GLOBALLY_UNIQUE);

	if (value == NULL) { // Default to true if clone-node-max > 1
	value = g_hash_table_lookup(rsc->priv->meta,
	PCMK_META_CLONE_NODE_MAX);
	if (value != NULL) {
	int node_max = 1;

	if ((pcmk__scan_min_int(value, &node_max, 0) == pcmk_rc_ok)
	&& (node_max > 1)) {
	return true;
	}
	}
	return false;
	}
	return crm_is_true(value);
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p scheduler is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (scheduler == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->priv->name != NULL)) {
	node_name = node->priv->name;
	}

	// Find the parameter table for given node
	if (rsc->priv->parameter_cache == NULL) {
	rsc->priv->parameter_cache = pcmk__strikey_table(free,
	free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->priv->parameter_cache,
	node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, scheduler);
	g_hash_table_insert(rsc->priv->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's \c PCMK_META_REQUIRES meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of \c PCMK_META_REQUIRES meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pcmk_resource_t rsc, const char value, bool is_default)
	{
	const pcmk_scheduler_t *scheduler = rsc->priv->scheduler;

	if (pcmk__str_eq(value, PCMK_VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK_VALUE_QUORUM, pcmk__str_casei)) {
	pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK_VALUE_FENCING, pcmk__str_casei)) {
	pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing);
	if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK_VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK_VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK_VALUE_QUORUM "\" because fencing is "
	"disabled", rsc->id);
	unpack_requires(rsc, PCMK_VALUE_QUORUM, true);
	return;

	} else {
	pcmk__set_rsc_flags(rsc, pcmk__rsc_needs_fencing
	\|pcmk__rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
	value = PCMK_VALUE_QUORUM;

	} else if (pcmk__is_primitive(rsc)
	&& xml_contains_remote_node(rsc->priv->xml)) {
	value = PCMK_VALUE_QUORUM;

	} else if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
	value = PCMK_VALUE_UNFENCING;

	} else if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	value = PCMK_VALUE_FENCING;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK_VALUE_NOTHING;

	} else {
	value = PCMK_VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	/*!
	* \internal
	* \brief Parse resource priority from meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	*/
	static void
	unpack_priority(pcmk_resource_t *rsc)
	{
	const char *value = g_hash_table_lookup(rsc->priv->meta,
	PCMK_META_PRIORITY);
	int rc = pcmk_parse_score(value, &(rsc->priv->priority), 0);

	if (rc != pcmk_rc_ok) {
	pcmk__config_warn("Using default (0) for resource %s "
	PCMK_META_PRIORITY
	" because '%s' is not a valid value: %s",
	rsc->id, value, pcmk_rc_str(rc));
	}
	}

	/*!
	* \internal
	* \brief Parse resource stickiness from meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	*/
	static void
	unpack_stickiness(pcmk_resource_t *rsc)
	{
	const char *value = g_hash_table_lookup(rsc->priv->meta,
	PCMK_META_RESOURCE_STICKINESS);

	if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) {
	// @COMPAT Deprecated since 2.1.8
	pcmk__config_warn("Support for setting "
	PCMK_META_RESOURCE_STICKINESS
	" to the explicit value '" PCMK_VALUE_DEFAULT
	"' is deprecated and will be removed in a "
	"future release (just leave it unset)");
	} else {
	int rc = pcmk_parse_score(value, &(rsc->priv->stickiness), 0);

	if (rc != pcmk_rc_ok) {
	pcmk__config_warn("Using default (0) for resource %s "
	PCMK_META_RESOURCE_STICKINESS
	" because '%s' is not a valid value: %s",
	rsc->id, value, pcmk_rc_str(rc));
	}
	}
	}

	/*!
	* \internal
	* \brief Parse resource migration threshold from meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	*/
	static void
	unpack_migration_threshold(pcmk_resource_t *rsc)
	{
	const char *value = g_hash_table_lookup(rsc->priv->meta,
	PCMK_META_MIGRATION_THRESHOLD);

	if (pcmk__str_eq(value, PCMK_VALUE_DEFAULT, pcmk__str_casei)) {
	// @COMPAT Deprecated since 2.1.8
	pcmk__config_warn("Support for setting "
	PCMK_META_MIGRATION_THRESHOLD
	" to the explicit value '" PCMK_VALUE_DEFAULT
	"' is deprecated and will be removed in a "
	"future release (just leave it unset)");
	rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY;
	} else {
	int rc = pcmk_parse_score(value, &(rsc->priv->ban_after_failures),
	PCMK_SCORE_INFINITY);

	if ((rc != pcmk_rc_ok) \|\| (rsc->priv->ban_after_failures < 0)) {
	pcmk__config_warn("Using default (" PCMK_VALUE_INFINITY
	") for resource %s meta-attribute "
	PCMK_META_MIGRATION_THRESHOLD
	" because '%s' is not a valid value: %s",
	rsc->id, value, pcmk_rc_str(rc));
	rsc->priv->ban_after_failures = PCMK_SCORE_INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pcmk_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pcmk_resource_t *rsc,
	pcmk_resource_t parent, pcmk_scheduler_t scheduler)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;
	pcmk__resource_private_t *rsc_private = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = scheduler->priv->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pcmk_resource_t));
	if (*rsc == NULL) {
	pcmk__sched_err(scheduler,
	"Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}

	(*rsc)->priv = calloc(1, sizeof(pcmk__resource_private_t));
	if ((*rsc)->priv == NULL) {
	pcmk__sched_err(scheduler,
	"Unable to allocate memory for resource '%s'", id);
	free(*rsc);
	return ENOMEM;
	}
	rsc_private = (*rsc)->priv;

	rsc_private->scheduler = scheduler;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	rsc_private->xml = expanded_xml;
	rsc_private->orig_xml = xml_obj;

	} else {
	rsc_private->xml = xml_obj;
	rsc_private->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	rsc_private->parent = parent;

	ops = pcmk__xe_first_child(rsc_private->xml, PCMK_XE_OPERATIONS, NULL,
	NULL);
	rsc_private->ops_xml = pcmk__xe_resolve_idref(ops, scheduler->input);

	rsc_private->variant = get_resource_type((const char *)
	rsc_private->xml->name);
	if (rsc_private->variant == pcmk__rsc_variant_unknown) {
	pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
	id, rsc_private->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	rsc_private->meta = pcmk__strkey_table(free, free);
	rsc_private->utilization = pcmk__strkey_table(free, free);
	rsc_private->probed_nodes = pcmk__strkey_table(NULL, free);
	rsc_private->allowed_nodes = pcmk__strkey_table(NULL, free);

	value = crm_element_value(rsc_private->xml, PCMK__META_CLONE);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	pcmk__insert_meta(rsc_private, PCMK__META_CLONE, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	rsc_private->fns = &resource_class_functions[rsc_private->variant];

	get_meta_attributes(rsc_private->meta, *rsc, NULL, scheduler);

	(*rsc)->flags = 0;
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_unassigned);

	if (!pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed);
	}

	rsc_private->orig_role = pcmk_role_stopped;
	rsc_private->next_role = pcmk_role_unknown;

	unpack_priority(*rsc);

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_critical);
	}

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_NOTIFY);
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_notify);
	}

	if (xml_contains_remote_node(rsc_private->xml)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_is_remote_connection);
	if (g_hash_table_lookup(rsc_private->meta, PCMK__META_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* PCMK_META_ALLOW_MIGRATE=false.
	*/
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_migratable);
	}

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_IS_MANAGED);
	if (value != NULL) {
	if (pcmk__str_eq(PCMK_VALUE_DEFAULT, value, pcmk__str_casei)) {
	// @COMPAT Deprecated since 2.1.8
	pcmk__config_warn("Support for setting " PCMK_META_IS_MANAGED
	" to the explicit value '" PCMK_VALUE_DEFAULT
	"' is deprecated and will be removed in a "
	"future release (just leave it unset)");
	} else if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_managed);
	} else {
	pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
	}
	}

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MAINTENANCE);
	if (crm_is_true(value)) {
	pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance);
	}
	if (pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
	pcmk__clear_rsc_flags(*rsc, pcmk__rsc_managed);
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_maintenance);
	}

	if (pcmk__is_clone(pe__const_top_resource(*rsc, false))) {
	if (detect_unique(*rsc)) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique);
	}
	if (crm_is_true(g_hash_table_lookup((*rsc)->priv->meta,
	PCMK_META_PROMOTABLE))) {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_promotable);
	}
	} else {
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_unique);
	}

	- // @COMPAT Deprecated meta-attribute
	- value = g_hash_table_lookup(rsc_private->meta, PCMK__META_RESTART_TYPE);
	- if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
	- // @COMPAT Not possible with schema validation enabled
	- rsc_private->restart_type = pcmk__restart_restart;
	- pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
	- (*rsc)->id);
	- pcmk__warn_once(pcmk__wo_restart_type,
	- "Support for " PCMK__META_RESTART_TYPE " is deprecated "
	- "and will be removed in a future release");
	-
	- } else {
	- rsc_private->restart_type = pcmk__restart_ignore;
	- pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
	- (*rsc)->id);
	- }
	-
	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_MULTIPLE_ACTIVE);
	if (pcmk__str_eq(value, PCMK_VALUE_STOP_ONLY, pcmk__str_casei)) {
	rsc_private->multiply_active_policy = pcmk__multiply_active_stop;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
	rsc_private->multiply_active_policy = pcmk__multiply_active_block;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, PCMK_VALUE_STOP_UNEXPECTED,
	pcmk__str_casei)) {
	rsc_private->multiply_active_policy = pcmk__multiply_active_unexpected;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // PCMK_VALUE_STOP_START
	if (!pcmk__str_eq(value, PCMK_VALUE_STOP_START,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_warn("%s is not a valid value for "
	PCMK_META_MULTIPLE_ACTIVE
	", using default of "
	"\"" PCMK_VALUE_STOP_START "\"",
	value);
	}
	rsc_private->multiply_active_policy = pcmk__multiply_active_restart;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: stop/start",
	(*rsc)->id);
	}

	unpack_stickiness(*rsc);
	unpack_migration_threshold(*rsc);

	if (pcmk__str_eq(crm_element_value(rsc_private->xml, PCMK_XA_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_fencing);
	pcmk__set_rsc_flags(*rsc, pcmk__rsc_fence_device);
	}

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup(rsc_private->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	pcmk_parse_interval_spec(value, &(rsc_private->failure_expiration_ms));
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, scheduler);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	pcmk_parse_interval_spec(value,
	&(rsc_private->remote_reconnect_ms));

	/* We want to override any default failure_timeout in use when remote
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use.
	*/
	rsc_private->failure_expiration_ms =
	rsc_private->remote_reconnect_ms;
	}
	}

	get_target_role(*rsc, &(rsc_private->next_role));
	pcmk__rsc_trace(rsc, "%s desired next state: %s", (rsc)->id,
	(rsc_private->next_role == pcmk_role_unknown)?
	"default" : pcmk_role_text(rsc_private->next_role));

	if (rsc_private->fns->unpack(*rsc, scheduler) == FALSE) {
	rsc_private->fns->free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default",
	scheduler);
	}

	pcmk__rsc_trace(rsc, "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk__rsc_notify)? "required" : "not required");

	pe__unpack_dataset_nvpairs(rsc_private->xml, PCMK_XE_UTILIZATION,
	&rule_data, rsc_private->utilization, NULL,
	scheduler);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, scheduler) == FALSE) {
	rsc_private->fns->free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pcmk_resource_t child, pcmk_resource_t rsc)
	{
	pcmk_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->priv->parent != NULL) {
	if (parent->priv->parent == rsc) {
	return TRUE;
	}
	parent = parent->priv->parent;
	}
	return FALSE;
	}

	pcmk_resource_t *
	uber_parent(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->priv->parent != NULL)
	&& !pcmk__is_bundle(parent->priv->parent)) {
	parent = parent->priv->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pcmk_resource_t *
	pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
	{
	const pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->priv->parent != NULL) {
	if (!include_bundle && pcmk__is_bundle(parent->priv->parent)) {
	break;
	}
	parent = parent->priv->parent;
	}
	return parent;
	}

	void
	common_free(pcmk_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	if (rsc->priv->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->priv->parameter_cache);
	}

	if ((rsc->priv->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {

	pcmk__xml_free(rsc->priv->xml);
	rsc->priv->xml = NULL;
	pcmk__xml_free(rsc->priv->orig_xml);
	rsc->priv->orig_xml = NULL;

	} else if (rsc->priv->orig_xml != NULL) {
	// rsc->private->xml was expanded from a template
	pcmk__xml_free(rsc->priv->xml);
	rsc->priv->xml = NULL;
	}
	free(rsc->id);

	free(rsc->priv->variant_opaque);
	free(rsc->priv->history_id);
	free(rsc->priv->pending_action);
	free(rsc->priv->assigned_node);

	g_list_free(rsc->priv->actions);
	g_list_free(rsc->priv->active_nodes);
	g_list_free(rsc->priv->launched);
	g_list_free(rsc->priv->dangling_migration_sources);
	g_list_free(rsc->priv->with_this_colocations);
	g_list_free(rsc->priv->this_with_colocations);
	g_list_free(rsc->priv->location_constraints);
	g_list_free(rsc->priv->ticket_constraints);

	if (rsc->priv->meta != NULL) {
	g_hash_table_destroy(rsc->priv->meta);
	}
	if (rsc->priv->utilization != NULL) {
	g_hash_table_destroy(rsc->priv->utilization);
	}
	if (rsc->priv->probed_nodes != NULL) {
	g_hash_table_destroy(rsc->priv->probed_nodes);
	}
	if (rsc->priv->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->priv->allowed_nodes);
	}

	free(rsc->priv);

	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->priv->partial_migration_source != NULL) {
	if (pcmk__same_node(node, rsc->priv->partial_migration_source)) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of pcmk__rsc_methods_t:active_node()
	static pcmk_node_t *
	active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->priv->active_nodes;
	iter != NULL; iter = iter->next) {

	if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to \c PCMK_META_REQUIRES
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* \c PCMK_META_REQUIRES meta-attribute.
	*/
	pcmk_node_t *
	pe__find_active_requires(const pcmk_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;
	}

	if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
	return rsc->priv->fns->active_node(rsc, count, NULL);
	} else {
	return rsc->priv->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pcmk_resource_t *rsc)
	{
	if (rsc->priv->children != NULL) {
	for (GList *item = rsc->priv->children;
	item != NULL; item = item->next) {
	pcmk_resource_t *child = item->data;

	child->priv->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
	\|\| (rsc->priv->orig_role > pcmk_role_stopped)) {
	rsc->priv->scheduler->priv->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->priv->scheduler->priv->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
	rsc->priv->scheduler->priv->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pcmk_resource_t rsc, enum rsc_role_e role, const char why)
	{
	pcmk__assert((rsc != NULL) && (why != NULL));
	if (rsc->priv->next_role != role) {
	pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, pcmk_role_text(rsc->priv->next_role),
	pcmk_role_text(role), why);
	rsc->priv->next_role = role;
	}
	}
	diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
	index 700bb7e646..8518fb0159 100644
	--- a/lib/pengine/pe_actions.c
	+++ b/lib/pengine/pe_actions.c
	@@ -1,1798 +1,1781 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml_internal.h>
	#include "pe_status_private.h"

	static void unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms);

	static void
	add_singleton(pcmk_scheduler_t scheduler, pcmk_action_t action)
	{
	if (scheduler->priv->singletons == NULL) {
	scheduler->priv->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(scheduler->priv->singletons, action->uuid, action);
	}

	static pcmk_action_t *
	lookup_singleton(pcmk_scheduler_t scheduler, const char action_uuid)
	{
	/* @TODO This is the only use of the pcmk_scheduler_t:singletons hash table.
	* Compare the performance of this approach to keeping the
	* pcmk_scheduler_t:actions list sorted by action key and just searching
	* that instead.
	*/
	if (scheduler->priv->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(scheduler->priv->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] scheduler Scheduler data
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pcmk_action_t *
	find_existing_action(const char key, const pcmk_resource_t rsc,
	const pcmk_node_t node, const pcmk_scheduler_t scheduler)
	{
	/* When rsc is NULL, it would be quicker to check
	* scheduler->priv->singletons, but checking all scheduler->priv->actions
	* takes the node into account.
	*/
	GList *actions = (rsc == NULL)? scheduler->priv->actions : rsc->priv->actions;
	GList *matches = find_actions(actions, key, node);
	pcmk_action_t *action = NULL;

	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	/*!
	* \internal
	* \brief Find the XML configuration corresponding to a specific action key
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] key "RSC_ACTION_INTERVAL" of action to find
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	static xmlNode *
	find_exact_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	const char *config_name = NULL;
	const char *interval_spec = NULL;
	guint tmp_ms = 0U;

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if (!include_disabled
	&& (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &tmp_ms);
	if (tmp_ms != interval_ms) {
	continue;
	}

	config_name = crm_element_value(operation, PCMK_XA_NAME);
	if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
	return operation;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Find the XML configuration of a resource action
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] action_name Action name to search for
	* \param[in] interval_ms Action interval (in milliseconds) to search for
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	xmlNode *
	pcmk__find_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	xmlNode *action_config = NULL;

	// Try requested action first
	action_config = find_exact_action_config(rsc, action_name, interval_ms,
	include_disabled);

	// For migrate_to and migrate_from actions, retry with "migrate"
	// @TODO This should be either documented or deprecated
	if ((action_config == NULL)
	&& pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	action_config = find_exact_action_config(rsc, "migrate", 0,
	include_disabled);
	}

	return action_config;
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pcmk_action_t *
	new_action(char key, const char task, pcmk_resource_t *rsc,
	const pcmk_node_t node, bool optional, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *action = pcmk__assert_alloc(1, sizeof(pcmk_action_t));

	action->rsc = rsc;
	action->task = pcmk__str_copy(task);
	action->uuid = key;
	action->scheduler = scheduler;

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pcmk__set_action_flags(action, pcmk__action_on_dc);
	}

	pcmk__set_action_flags(action, pcmk__action_runnable);
	if (optional) {
	pcmk__set_action_flags(action, pcmk__action_optional);
	} else {
	pcmk__clear_action_flags(action, pcmk__action_optional);
	}

	if (rsc == NULL) {
	action->meta = pcmk__strkey_table(free, free);
	} else {
	guint interval_ms = 0;

	parse_op_key(key, NULL, NULL, &interval_ms);
	action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
	true);

	/* If the given key is for one of the many notification pseudo-actions
	* (pre_notify_promote, etc.), the actual action name is "notify"
	*/
	if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
	action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
	0, true);
	}

	unpack_operation(action, action->op_entry, interval_ms);
	}

	pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	scheduler->priv->next_action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	pcmk__node_name(node));
	action->id = scheduler->priv->next_action_id++;

	scheduler->priv->actions = g_list_prepend(scheduler->priv->actions, action);
	if (rsc == NULL) {
	add_singleton(scheduler, action);
	} else {
	rsc->priv->actions = g_list_prepend(rsc->priv->actions, action);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Unpack a resource's action-specific instance parameters
	*
	* \param[in] action_xml XML of action's configuration in CIB (if any)
	* \param[in,out] node_attrs Table of node attributes (for rule evaluation)
	* \param[in,out] scheduler Cluster working set (for rule evaluation)
	*
	* \return Newly allocated hash table of action-specific instance parameters
	*/
	GHashTable *
	pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = node_attrs,
	.now = scheduler->priv->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__unpack_dataset_nvpairs(action_xml, PCMK_XE_INSTANCE_ATTRIBUTES,
	&rule_data, params, NULL, scheduler);
	return params;
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in,out] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pcmk_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pcmk__action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pcmk__rsc_managed)
	&& (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is optional (%s is unmanaged)",
	action->uuid, pcmk__node_name(action->node),
	action->rsc->id);
	pcmk__set_action_flags(action, pcmk__action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pcmk__clear_action_flags(action, pcmk__action_optional);
	}
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	enum pe_quorum_policy policy = scheduler->no_quorum_policy;

	if (pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
	policy = pcmk_no_quorum_ignore;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
	switch (rsc->priv->orig_role) {
	case pcmk_role_promoted:
	case pcmk_role_unpromoted:
	if (rsc->priv->next_role > pcmk_role_unpromoted) {
	pe__set_next_role(rsc, pcmk_role_unpromoted,
	PCMK_OPT_NO_QUORUM_POLICY "=demote");
	}
	policy = pcmk_no_quorum_ignore;
	break;
	default:
	policy = pcmk_no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in,out] action Action to update
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pcmk_action_t *action,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = action->rsc;

	if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pcmk__rsc_trace(rsc, "%s is unrunnable (unallocated)", action->uuid);
	pcmk__clear_action_flags(action, pcmk__action_runnable);

	} else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
	&& !(action->node->details->online)
	&& (!pcmk__is_guest_or_bundle_node(action->node)
	\|\| pcmk_is_set(action->node->priv->flags,
	pcmk__node_remote_reset))) {
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
	action->uuid, pcmk__node_name(action->node));
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
	&& action->node->details->pending) {
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	do_crm_log(LOG_WARNING,
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, pcmk__node_name(action->node));

	} else if (action->needs == pcmk__requires_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pcmk__is_guest_or_bundle_node(action->node)
	&& !pe_can_fence(scheduler, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pcmk__rsc_debug(rsc,
	"%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	} else {
	pcmk__rsc_trace(rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, pcmk__node_name(action->node));
	pcmk__set_action_flags(action, pcmk__action_runnable);
	}

	} else {
	switch (effective_quorum_policy(rsc, scheduler)) {
	case pcmk_no_quorum_stop:
	pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid, pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case pcmk_no_quorum_freeze:
	if (!rsc->priv->fns->active(rsc, TRUE)
	\|\| (rsc->priv->next_role > rsc->priv->orig_role)) {
	pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid,
	pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pcmk__set_action_flags(action, pcmk__action_runnable);
	break;
	}
	}
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value,
	PCMK_VALUE_STANDBY, PCMK_VALUE_DEMOTE,
	PCMK_VALUE_STOP, NULL);
	}

	/*!
	* \internal
	* \brief Validate (and possibly reset) resource action's on_fail meta-attribute
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] action_config Action configuration XML from CIB (if any)
	* \param[in,out] meta Table of action meta-attributes
	*/
	static void
	validate_on_fail(const pcmk_resource_t rsc, const char action_name,
	const xmlNode action_config, GHashTable meta)
	{
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);
	guint interval_ms = 0U;

	// Stop actions can only use certain on-fail values
	if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", rsc->id, value);
	g_hash_table_remove(meta, PCMK_META_ON_FAIL);
	return;
	}

	/* Demote actions default on-fail to the on-fail value for the first
	* recurring monitor for the promoted role (if any).
	*/
	if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
	&& (value == NULL)) {

	/* @TODO This does not consider promote options set in a meta-attribute
	* block (which may have rules that need to be evaluated) rather than
	* XML properties.
	*/
	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	const char *promote_on_fail = NULL;

	/* We only care about explicit on-fail (if promote uses default, so
	* can demote)
	*/
	promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL);
	if (promote_on_fail == NULL) {
	continue;
	}

	// We only care about recurring monitors for the promoted role
	name = crm_element_value(operation, PCMK_XA_NAME);
	role = crm_element_value(operation, PCMK_XA_ROLE);
	if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
	continue;
	}
	interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// We only care about enabled monitors
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	/* Demote actions can't default to
	* PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
	*/
	if (pcmk__str_eq(promote_on_fail, PCMK_VALUE_DEMOTE,
	pcmk__str_casei)) {
	continue;
	}

	// Use value from first applicable promote action found
	pcmk__insert_dup(meta, PCMK_META_ON_FAIL, promote_on_fail);
	}
	return;
	}

	if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
	&& !pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {

	pcmk__insert_dup(meta, PCMK_META_ON_FAIL, PCMK_VALUE_IGNORE);
	return;
	}

	// PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE is allowed only for certain actions
	if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
	name = crm_element_value(action_config, PCMK_XA_NAME);
	role = crm_element_value(action_config, PCMK_XA_ROLE);
	interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);

	if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
	&& ((interval_ms == 0U)
	\|\| !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL))) {

	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", rsc->id, name);
	g_hash_table_remove(meta, PCMK_META_ON_FAIL);
	return;
	}
	}
	}

	static int
	unpack_timeout(const char *value)
	{
	long long timeout_ms = crm_get_msec(value);

	if (timeout_ms <= 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}
	return (int) QB_MIN(timeout_ms, INT_MAX);
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, const xmlNode xml_obj,
	guint interval_ms, const crm_time_t *now,
	long long *start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for "
	"operation '%s' because '%s' is not valid",
	pcmk__s(pcmk__xe_id(xml_obj), "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result, pcmk__s(pcmk__xe_id(xml_obj), "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	long long start_delay_ms = 0;

	if (value == NULL) {
	return 0;
	}

	start_delay_ms = crm_get_msec(value);
	start_delay_ms = QB_MIN(start_delay_ms, INT_MAX);
	if (start_delay_ms < 0) {
	start_delay_ms = 0;
	}

	if (meta != NULL) {
	g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY),
	pcmk__itoa(start_delay_ms));
	}

	return (int) start_delay_ms;
	}

	/*!
	* \internal
	* \brief Find a resource's most frequent recurring monitor
	*
	* \param[in] rsc Resource to check
	*
	* \return Operation XML configured for most frequent recurring monitor for
	* \p rsc (if any)
	*/
	static xmlNode *
	most_frequent_monitor(const pcmk_resource_t *rsc)
	{
	guint min_interval_ms = G_MAXUINT;
	xmlNode *op = NULL;

	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	guint interval_ms = 0U;
	const char *interval_spec = crm_element_value(operation,
	PCMK_META_INTERVAL);

	// We only care about enabled recurring monitors
	if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
	PCMK_ACTION_MONITOR, pcmk__str_none)) {
	continue;
	}

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	if (interval_ms < min_interval_ms) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	return op;
	}

	/*!
	* \internal
	* \brief Unpack action meta-attributes
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node that action is on
	* \param[in] action_name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] action_config Action XML configuration from CIB (if any)
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds) from its CIB XML
	* configuration (including defaults).
	*
	* \return Newly allocated hash table with normalized action meta-attributes
	*/
	GHashTable *
	pcmk__unpack_action_meta(pcmk_resource_t rsc, const pcmk_node_t node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config)
	{
	GHashTable *meta = NULL;
	const char *timeout_spec = NULL;
	const char *str = NULL;

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
	.provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
	.agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE),
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action_name,
	.interval = interval_ms,
	};

	pe_rule_eval_data_t rule_data = {
	/* Node attributes are not set because node expressions are not allowed
	* for meta-attributes
	*/
	.now = rsc->priv->scheduler->priv->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data,
	};

	meta = pcmk__strkey_table(free, free);

	if (action_config != NULL) {
	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(action_config, PCMK_XE_META_ATTRIBUTES,
	&rule_data, meta, NULL,
	rsc->priv->scheduler);

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	* (See below for the only exception, fence device start/probe timeout.)
	*/
	for (xmlAttrPtr attr = action_config->properties;
	attr != NULL; attr = attr->next) {
	pcmk__insert_dup(meta, (const char *) attr->name,
	pcmk__xml_attr_value(attr));
	}
	}

	// Derive default timeout for probes from recurring monitor timeouts
	if (pcmk_is_probe(action_name, interval_ms)
	&& (g_hash_table_lookup(meta, PCMK_META_TIMEOUT) == NULL)) {

	xmlNode *min_interval_mon = most_frequent_monitor(rsc);

	if (min_interval_mon != NULL) {
	/* @TODO This does not consider timeouts set in
	* PCMK_XE_META_ATTRIBUTES blocks (which may also have rules that
	* need to be evaluated).
	*/
	timeout_spec = crm_element_value(min_interval_mon,
	PCMK_META_TIMEOUT);
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting default timeout for %s probe to "
	"most frequent monitor's timeout '%s'",
	rsc->id, timeout_spec);
	pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
	}
	}
	}

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(rsc->priv->scheduler->priv->op_defaults,
	PCMK_XE_META_ATTRIBUTES, &rule_data, meta, NULL,
	rsc->priv->scheduler);

	g_hash_table_remove(meta, PCMK_XA_ID);

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_INTERVAL),
	crm_strdup_printf("%u", interval_ms));
	} else {
	g_hash_table_remove(meta, PCMK_META_INTERVAL);
	}

	/* Timeout order of precedence (highest to lowest):
	* 1. pcmk_monitor_timeout resource parameter (only for starts and probes
	* when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
	* monitors via the executor instead)
	* 2. timeout configured in <op> (with <op timeout> taking precedence over
	* <op> <meta_attributes>)
	* 3. timeout configured in <op_defaults> <meta_attributes>
	* 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
	*/

	// Check for pcmk_monitor_timeout
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
	\|\| pcmk_is_probe(action_name, interval_ms))) {

	GHashTable *params = pe_rsc_params(rsc, node, rsc->priv->scheduler);

	timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting timeout for %s %s to "
	"pcmk_monitor_timeout (%s)",
	rsc->id, action_name, timeout_spec);
	pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
	}
	}

	// Normalize timeout to positive milliseconds
	timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT);
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_TIMEOUT),
	pcmk__itoa(unpack_timeout(timeout_spec)));

	// Ensure on-fail has a valid value
	validate_on_fail(rsc, action_name, action_config, meta);

	// Normalize PCMK_META_START_DELAY
	str = g_hash_table_lookup(meta, PCMK_META_START_DELAY);
	if (str != NULL) {
	unpack_start_delay(str, meta);
	} else {
	long long start_delay = 0;

	str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN);
	if (unpack_interval_origin(str, action_config, interval_ms,
	rsc->priv->scheduler->priv->now,
	&start_delay)) {
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_START_DELAY),
	crm_strdup_printf("%lld", start_delay));
	}
	}
	return meta;
	}

	/*!
	* \internal
	* \brief Determine an action's quorum and fencing dependency
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action being unpacked
	*
	* \return Quorum and fencing dependency appropriate to action
	*/
	enum pcmk__requires
	pcmk__action_requires(const pcmk_resource_t rsc, const char action_name)
	{
	const char *value = NULL;
	enum pcmk__requires requires = pcmk__requires_nothing;

	CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);

	if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
	requires = pcmk__requires_fencing;
	value = "fencing";

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_quorum)) {
	requires = pcmk__requires_quorum;
	value = "quorum";

	} else {
	value = "nothing";
	}
	pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
	return requires;
	}

	/*!
	* \internal
	* \brief Parse action failure response from a user-provided string
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] value User-provided configuration value for on-fail
	*
	* \return Action failure response parsed from \p text
	*/
	enum pcmk__on_fail
	pcmk__parse_on_fail(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, const char *value)
	{
	const char *desc = NULL;
	bool needs_remote_reset = false;
	enum pcmk__on_fail on_fail = pcmk__on_fail_ignore;
	const pcmk_scheduler_t *scheduler = NULL;

	// There's no enum value for unknown or invalid, so assert
	pcmk__assert((rsc != NULL) && (action_name != NULL));
	scheduler = rsc->priv->scheduler;

	if (value == NULL) {
	// Use default

	} else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_block;
	desc = "block";

	} else if (pcmk__str_eq(value, PCMK_VALUE_FENCE, pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	on_fail = pcmk__on_fail_fence_node;
	desc = "node fencing";
	} else {
	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for "
	"%s of %s to 'stop' because 'fence' is not "
	"valid when fencing is disabled",
	action_name, rsc->id);
	on_fail = pcmk__on_fail_stop;
	desc = "stop resource";
	}

	} else if (pcmk__str_eq(value, PCMK_VALUE_STANDBY, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_standby_node;
	desc = "node standby";

	} else if (pcmk__strcase_any_of(value,
	PCMK_VALUE_IGNORE, PCMK_VALUE_NOTHING,
	NULL)) {
	desc = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	on_fail = pcmk__on_fail_ban;
	desc = "force migration";

	} else if (pcmk__str_eq(value, PCMK_VALUE_STOP, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_stop;
	desc = "stop resource";

	} else if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_restart;
	desc = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, PCMK_VALUE_RESTART_CONTAINER,
	pcmk__str_casei)) {
	if (rsc->priv->launcher == NULL) {
	pcmk__rsc_debug(rsc,
	"Using default " PCMK_META_ON_FAIL " for %s "
	"of %s because it does not have a launcher",
	action_name, rsc->id);
	} else {
	on_fail = pcmk__on_fail_restart_container;
	desc = "restart container (and possibly migrate)";
	}

	} else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_demote;
	desc = "demote instance";

	} else {
	pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for "
	"%s of %s because '%s' is not valid",
	action_name, rsc->id, value);
	}

	/* Remote node connections are handled specially. Failures that result
	* in dropping an active connection must result in fencing. The only
	* failures that don't are probes and starts. The user can explicitly set
	* PCMK_META_ON_FAIL=PCMK_VALUE_FENCE to fence after start failures.
	*/
	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
	&& pcmk__is_remote_node(pcmk_find_node(scheduler, rsc->id))
	&& !pcmk_is_probe(action_name, interval_ms)
	&& !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
	needs_remote_reset = true;
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	desc = NULL; // Force default for unmanaged connections
	}
	}

	if (desc != NULL) {
	// Explicit value used, default not needed

	} else if (rsc->priv->launcher != NULL) {
	on_fail = pcmk__on_fail_restart_container;
	desc = "restart container (and possibly migrate) (default)";

	} else if (needs_remote_reset) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	desc = "fence remote node (default)";
	} else {
	desc = "recover remote node connection (default)";
	}
	on_fail = pcmk__on_fail_reset_remote;
	} else {
	on_fail = pcmk__on_fail_stop;
	desc = "stop unmanaged remote node (enforcing default)";
	}

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	on_fail = pcmk__on_fail_fence_node;
	desc = "resource fence (default)";
	} else {
	on_fail = pcmk__on_fail_block;
	desc = "resource block (default)";
	}

	} else {
	on_fail = pcmk__on_fail_restart;
	desc = "restart (and possibly migrate) (default)";
	}

	pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
	pcmk__readable_interval(interval_ms), action_name,
	rsc->id, desc);
	return on_fail;
	}

	/*!
	* \internal
	* \brief Determine a resource's role after failure of an action
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] on_fail Failure handling for action
	* \param[in] meta Unpacked action meta-attributes
	*
	* \return Resource role that results from failure of action
	*/
	enum rsc_role_e
	pcmk__role_after_failure(const pcmk_resource_t rsc, const char action_name,
	enum pcmk__on_fail on_fail, GHashTable *meta)
	{
	- const char *value = NULL;
	enum rsc_role_e role = pcmk_role_unknown;

	// Set default for role after failure specially in certain circumstances
	switch (on_fail) {
	case pcmk__on_fail_stop:
	role = pcmk_role_stopped;
	break;

	case pcmk__on_fail_reset_remote:
	if (rsc->priv->remote_reconnect_ms != 0U) {
	role = pcmk_role_stopped;
	}
	break;

	default:
	break;
	}

	- // @COMPAT Check for explicitly configured role (deprecated)
	- value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE);
	- if (value != NULL) {
	- pcmk__warn_once(pcmk__wo_role_after,
	- "Support for " PCMK__META_ROLE_AFTER_FAILURE " is "
	- "deprecated and will be removed in a future release");
	- if (role == pcmk_role_unknown) {
	- role = pcmk_parse_role(value);
	- if (role == pcmk_role_unknown) {
	- pcmk__config_err("Ignoring invalid value %s for "
	- PCMK__META_ROLE_AFTER_FAILURE,
	- value);
	- }
	- }
	- }
	-
	if (role == pcmk_role_unknown) {
	// Use default
	if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	role = pcmk_role_unpromoted;
	} else {
	role = pcmk_role_started;
	}
	}
	pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
	rsc->id, action_name, pcmk_role_text(role));
	return role;
	}

	/*!
	* \internal
	* \brief Unpack action configuration
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds), requirements, and
	* failure policy from its CIB XML configuration (including defaults).
	*
	* \param[in,out] action Resource action to unpack into
	* \param[in] xml_obj Action configuration XML (NULL for defaults only)
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms)
	{
	const char *value = NULL;

	action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
	action->task, interval_ms, xml_obj);
	action->needs = pcmk__action_requires(action->rsc, action->task);

	value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL);
	action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
	interval_ms, value);

	action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
	action->on_fail, action->meta);
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in,out] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Action object corresponding to arguments (guaranteed not to be
	* \c NULL)
	* \note This function takes ownership of (and might free) \p key, and
	* \p scheduler takes ownership of the returned action (the caller should
	* not free it).
	*/
	pcmk_action_t *
	custom_action(pcmk_resource_t rsc, char key, const char *task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	pcmk__assert((key != NULL) && (task != NULL) && (scheduler != NULL));

	action = find_existing_action(key, rsc, on_node, scheduler);
	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, scheduler);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	/* An action can be initially created with a NULL node, and later have
	* the node added via find_existing_action() (above) -> find_actions().
	* That is why the extra parameters are unpacked here rather than in
	* new_action().
	*/
	if ((action->node != NULL) && (action->op_entry != NULL)
	&& !pcmk_is_set(action->flags, pcmk__action_attrs_evaluated)) {

	GHashTable *attrs = action->node->priv->attrs;

	if (action->extra != NULL) {
	g_hash_table_destroy(action->extra);
	}
	action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
	attrs, scheduler);
	pcmk__set_action_flags(action, pcmk__action_attrs_evaluated);
	}

	update_resource_action_runnable(action, scheduler);
	}

	if (action->extra == NULL) {
	action->extra = pcmk__strkey_table(free, free);
	}

	return action;
	}

	pcmk_action_t *
	get_pseudo_op(const char name, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *op = lookup_singleton(scheduler, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
	pcmk__set_action_flags(op, pcmk__action_pseudo\|pcmk__action_runnable);
	}
	return op;
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *candidate = gIter->data;

	if (candidate->priv->children != NULL) {
	matches = find_unfencing_devices(candidate->priv->children,
	matches);

	} else if (!pcmk_is_set(candidate->flags, pcmk__rsc_fence_device)) {
	continue;

	} else if (pcmk_is_set(candidate->flags, pcmk__rsc_needs_unfencing)) {
	matches = g_list_prepend(matches, candidate);

	} else if (pcmk__str_eq(g_hash_table_lookup(candidate->priv->meta,
	PCMK_STONITH_PROVIDES),
	PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(const pcmk_node_t *node,
	const pcmk_scheduler_t *scheduler)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
	if (scheduler->priv->priority_fencing_ms == 0U) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->priv->variant != pcmk__node_variant_cluster) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *n = gIter->data;

	if (n->priv->variant != pcmk__node_variant_cluster) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->priv->priority > top_priority) {
	top_priority = n->priv->priority;
	}

	if (member_count == 1
	\|\| n->priv->priority < lowest_priority) {
	lowest_priority = n->priv->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->priv->priority < top_priority) {
	return 0;
	}

	return (int) (scheduler->priv->priority_fencing_ms / 1000U);
	}

	pcmk_action_t *
	pe_fence_op(pcmk_node_t node, const char op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler)
	{
	char *op_key = NULL;
	pcmk_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = scheduler->priv->fence_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s",
	PCMK_ACTION_STONITH, node->priv->name, op);

	stonith_op = lookup_singleton(scheduler, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
	TRUE, scheduler);

	pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE, node->priv->name);
	pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE_UUID,
	node->priv->id);
	pcmk__insert_meta(stonith_op, PCMK__META_STONITH_ACTION, op);

	if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
	/* Extra work to detect device changes
	*/
	GString *digests_all = g_string_sized_new(1024);
	GString *digests_secure = g_string_sized_new(1024);

	GList *matches = find_unfencing_devices(scheduler->priv->resources,
	NULL);

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->priv->meta,
	PCMK_XA_TYPE);
	pcmk__op_digest_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node,
	scheduler);
	if (data->rc == pcmk__digest_mismatch) {
	optional = FALSE;
	crm_notice("Unfencing node %s because the definition of "
	"%s changed", pcmk__node_name(node), match->id);
	if (!pcmk__is_daemon && (scheduler->priv->out != NULL)) {
	pcmk__output_t *out = scheduler->priv->out;

	out->info(out,
	"notice: Unfencing node %s because the "
	"definition of %s changed",
	pcmk__node_name(node), match->id);
	}
	}

	pcmk__g_strcat(digests_all,
	match->id, ":", agent, ":",
	data->digest_all_calc, ",", NULL);
	pcmk__g_strcat(digests_secure,
	match->id, ":", agent, ":",
	data->digest_secure_calc, ",", NULL);
	}
	pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_ALL,
	digests_all->str);
	g_string_free(digests_all, TRUE);

	pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_SECURE,
	digests_secure->str);
	g_string_free(digests_secure, TRUE);

	g_list_free(matches);
	}

	} else {
	free(op_key);
	}

	if ((scheduler->priv->priority_fencing_ms > 0U)

	/* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
	* applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
	* an indicator.
	*/
	&& (priority_delay

	/* The priority delay needs to be recalculated if this function has
	* been called by schedule_fencing_and_shutdowns() after node
	* priority has already been calculated by native_add_running().
	*/
	\|\| g_hash_table_lookup(stonith_op->meta,
	PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
	* it's 0 for the targeting node. So that it takes precedence over
	* any possible `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
	scheduler));

	g_hash_table_insert(stonith_op->meta,
	strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(scheduler, node)) {
	pcmk__clear_action_flags(stonith_op, pcmk__action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	pe_free_action(pcmk_action_t *action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free);
	g_list_free_full(action->actions_after, free);
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	enum pcmk__action_type
	get_complex_task(const pcmk_resource_t rsc, const char name)
	{
	enum pcmk__action_type task = pcmk__parse_action(name);

	if (pcmk__is_primitive(rsc)) {
	switch (task) {
	case pcmk__action_stopped:
	case pcmk__action_started:
	case pcmk__action_demoted:
	case pcmk__action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s",
	name, rsc->id);
	--task;
	break;
	default:
	break;
	}
	}
	return task;
	}

	/*!
	* \internal
	* \brief Find first matching action in a list
	*
	* \param[in] input List of actions to search
	* \param[in] uuid If not NULL, action must have this UUID
	* \param[in] task If not NULL, action must have this action name
	* \param[in] on_node If not NULL, action must be on this node
	*
	* \return First action in list that matches criteria, or NULL if none
	*/
	pcmk_action_t *
	find_first_action(const GList input, const char uuid, const char *task,
	const pcmk_node_t *on_node)
	{
	CRM_CHECK(uuid \|\| task, return NULL);

	for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (pcmk__same_node(on_node, action->node)) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, pcmk__node_name(on_node));

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (pcmk__same_node(on_node, action->node)) {
	crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__same_node(on_node, action->node)) {

	crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->priv->actions, key, node);
	} else {
	result = find_actions(rsc->priv->actions, key, node);
	}
	free(key);
	return result;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(const pcmk_action_t *action, enum pcmk__action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pcmk__action_runnable:
	change = "unrunnable";
	break;
	case pcmk__action_migratable:
	change = "unmigrateable";
	break;
	case pcmk__action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason,
	pcmk__s(reason, "(none)"));
	} else if (action->reason == NULL) {
	pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, pcmk__s(reason, "(none)"));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in,out] rsc Resource to clear
	* \param[in] node Node to clear history on
	*/
	void
	pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__assert((rsc != NULL) && (node != NULL));

	custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->priv->scheduler);
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	int
	pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
	const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);

	const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE);
	const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE);
	bool same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);

	if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
	/* We have duplicate PCMK__XE_LRM_RSC_OP entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pcmk__config_err("Duplicate " PCMK__XE_LRM_RSC_OP " entries named %s",
	a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, PCMK__XA_CALL_ID, &a_call_id);
	crm_element_value_int(xml_b, PCMK__XA_CALL_ID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (a_call_id >= 0 && b_call_id >= 0
	&& (!same_node \|\| a_call_id == b_call_id)) {
	/* The op and last_failed_op are the same. Order on
	* PCMK_XA_LAST_RC_CHANGE.
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, PCMK_XA_LAST_RC_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, PCMK_XA_LAST_RC_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation.
	* Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative
	* to the other.
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a,
	PCMK__XA_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b,
	PCMK__XA_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));
	}

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	return pe__is_newer_op((const xmlNode ) a, (const xmlNode ) b);
	}

	/*!
	* \internal
	* \brief Create a new pseudo-action for a resource
	*
	* \param[in,out] rsc Resource to create action for
	* \param[in] task Action name
	* \param[in] optional Whether action should be considered optional
	* \param[in] runnable Whethe action should be considered runnable
	*
	* \return New action object corresponding to arguments
	*/
	pcmk_action_t *
	pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char task, bool optional,
	bool runnable)
	{
	pcmk_action_t *action = NULL;

	pcmk__assert((rsc != NULL) && (task != NULL));

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
	optional, rsc->priv->scheduler);
	pcmk__set_action_flags(action, pcmk__action_pseudo);
	if (runnable) {
	pcmk__set_action_flags(action, pcmk__action_runnable);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Add the expected result to an action
	*
	* \param[in,out] action Action to add expected result to
	* \param[in] expected_result Expected result to add
	*
	* \note This is more efficient than calling pcmk__insert_meta().
	*/
	void
	pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
	{
	pcmk__assert((action != NULL) && (action->meta != NULL));

	g_hash_table_insert(action->meta, pcmk__str_copy(PCMK__META_OP_TARGET_RC),
	pcmk__itoa(expected_result));
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 7:27 PM (5 h, 49 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665548
Default Alt Text: (239 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions