No OneTemporary
Actions

Size

99 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/common/scheduler.h b/include/crm/common/scheduler.h
	index ed44ee4cf1..603c0858c2 100644
	--- a/include/crm/common/scheduler.h
	+++ b/include/crm/common/scheduler.h
	@@ -1,138 +1,137 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_SCHEDULER__H
	#define PCMK__CRM_COMMON_SCHEDULER__H

	#include <sys/types.h> // time_t
	#include <libxml/tree.h> // xmlNode
	#include <glib.h> // guint, GList, GHashTable

	#include <crm/common/iso8601.h> // crm_time_t

	#include <crm/common/actions.h>
	#include <crm/common/nodes.h>
	#include <crm/common/resources.h>
	#include <crm/common/roles.h>
	#include <crm/common/rules.h>
	#include <crm/common/scheduler_types.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Scheduler API
	* \ingroup core
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this enum
	//! Possible responses to loss of quorum
	enum pe_quorum_policy {
	pcmk_no_quorum_freeze, //!< Do not recover resources from outside partition
	pcmk_no_quorum_stop, //!< Stop all resources in partition
	pcmk_no_quorum_ignore, //!< Act as if partition still holds quorum
	pcmk_no_quorum_fence, //!< Fence all nodes in partition
	pcmk_no_quorum_demote, //!< Demote promotable resources and stop all others

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	// NOTE: sbd (as of at least 1.5.2) uses this value
	//! \deprecated Use pcmk_no_quorum_freeze instead
	no_quorum_freeze = pcmk_no_quorum_freeze,

	// NOTE: sbd (as of at least 1.5.2) uses this value
	//! \deprecated Use pcmk_no_quorum_stop instead
	no_quorum_stop = pcmk_no_quorum_stop,

	// NOTE: sbd (as of at least 1.5.2) uses this value
	//! \deprecated Use pcmk_no_quorum_ignore instead
	no_quorum_ignore = pcmk_no_quorum_ignore,

	//! \deprecated Use pcmk_no_quorum_fence instead
	no_quorum_suicide = pcmk_no_quorum_fence,

	// NOTE: sbd (as of at least 1.5.2) uses this value
	//! \deprecated Use pcmk_no_quorum_demote instead
	no_quorum_demote = pcmk_no_quorum_demote,
	#endif
	};

	//! \internal Do not use
	typedef struct pcmk__scheduler_private pcmk__scheduler_private_t;

	/* Implementation of pcmk_scheduler_t
	*
	* @COMPAT Drop this struct once all members are moved to
	* pcmk__scheduler_private_t, and repoint pcmk_scheduler_t to that
	*/
	//!@{
	//! \deprecated Do not use (public access will be removed in a future release)
	struct pcmk__scheduler {
	// Be careful about when each piece of information is available and final

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Set scheduler input with pcmk_set_scheduler_cib() instead
	xmlNode *input; // CIB XML

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_get_dc() instead
	pcmk_node_t *dc_node; // Node object for DC

	// NOTE: sbd (as of at least 1.5.2) uses this
	// @COMPAT Change to uint64_t at a compatibility break
	//! \deprecated Call pcmk_has_quorum() to check quorum
	unsigned long long flags; // Group of enum pcmk__scheduler_flags

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_get_no_quorum_policy() to get no-quorum policy
	enum pe_quorum_policy no_quorum_policy; // Response to loss of quorum

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call pcmk_find_node() to find a node instead
	GList *nodes; // Nodes in cluster

	- int action_id; // ID to use for next created action
	xmlNode *graph; // Transition graph
	GHashTable *template_rsc_sets; // Mappings of template ID to resource ID

	// @COMPAT Replace this with a fencer variable (only place it's used)
	const char *localhost; // \deprecated Do not use

	GHashTable tags; // Configuration tags (ID -> pcmk__idref_t)
	int blocked_resources; // Number of blocked resources in cluster
	int disabled_resources; // Number of disabled resources in cluster
	GList *param_check; // History entries that need to be checked
	GList *stop_needed; // Containers that need stop actions
	time_t recheck_by; // Hint to controller when to reschedule
	int ninstances; // Total number of resource instances
	guint shutdown_lock; // How long to lock resources (seconds)
	int priority_fencing_delay; // Priority fencing delay

	pcmk__scheduler_private_t *priv; // For Pacemaker use only

	guint node_pending_timeout; // Pending join times out after this (ms)
	};
	//!@}

	pcmk_node_t pcmk_get_dc(const pcmk_scheduler_t scheduler);
	enum pe_quorum_policy pcmk_get_no_quorum_policy(const pcmk_scheduler_t
	*scheduler);

	int pcmk_set_scheduler_cib(pcmk_scheduler_t scheduler, xmlNode cib);

	bool pcmk_has_quorum(const pcmk_scheduler_t *scheduler);
	pcmk_node_t pcmk_find_node(const pcmk_scheduler_t scheduler,
	const char *node_name);

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_SCHEDULER__H
	diff --git a/include/crm/common/scheduler_internal.h b/include/crm/common/scheduler_internal.h
	index e47bf35d8f..fe478dfd18 100644
	--- a/include/crm/common/scheduler_internal.h
	+++ b/include/crm/common/scheduler_internal.h
	@@ -1,275 +1,276 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
	#define PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H

	#include <crm/common/action_relation_internal.h>
	#include <crm/common/actions_internal.h>
	#include <crm/common/attrs_internal.h>
	#include <crm/common/bundles_internal.h>
	#include <crm/common/clone_internal.h>
	#include <crm/common/digest_internal.h>
	#include <crm/common/failcounts_internal.h>
	#include <crm/common/group_internal.h>
	#include <crm/common/history_internal.h>
	#include <crm/common/location_internal.h>
	#include <crm/common/nodes_internal.h>
	#include <crm/common/primitive_internal.h>
	#include <crm/common/remote_internal.h>
	#include <crm/common/resources_internal.h>
	#include <crm/common/roles_internal.h>
	#include <crm/common/rules_internal.h>
	#include <crm/common/tickets_internal.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	enum pcmk__check_parameters {
	/* Clear fail count if parameters changed for un-expired start or monitor
	* last_failure.
	*/
	pcmk__check_last_failure,

	/* Clear fail count if parameters changed for start, monitor, promote, or
	* migrate_from actions for active resources.
	*/
	pcmk__check_active,
	};

	// Scheduling options and conditions
	enum pcmk__scheduler_flags {
	// No scheduler flags set (compare with equality rather than bit set)
	pcmk__sched_none = 0ULL,

	/* These flags are dynamically determined conditions */

	// Whether partition has quorum (via \c PCMK_XA_HAVE_QUORUM attribute)
	//! \deprecated Call pcmk_has_quorum() to check quorum instead
	pcmk__sched_quorate = (1ULL << 0),

	// Whether cluster is symmetric (via symmetric-cluster property)
	pcmk__sched_symmetric_cluster = (1ULL << 1),

	// Whether scheduling encountered a non-configuration error
	pcmk__sched_processing_error = (1ULL << 2),

	// Whether cluster is in maintenance mode (via maintenance-mode property)
	pcmk__sched_in_maintenance = (1ULL << 3),

	// Whether fencing is enabled (via stonith-enabled property)
	pcmk__sched_fencing_enabled = (1ULL << 4),

	// Whether cluster has a fencing resource (via CIB resources)
	/*! \deprecated To indicate the cluster has a fencing resource, add either a
	* fencing resource configuration or the have-watchdog cluster option to the
	* input CIB
	*/
	pcmk__sched_have_fencing = (1ULL << 5),

	// Whether any resource provides or requires unfencing (via CIB resources)
	pcmk__sched_enable_unfencing = (1ULL << 6),

	// Whether concurrent fencing is allowed (via concurrent-fencing property)
	pcmk__sched_concurrent_fencing = (1ULL << 7),

	/*
	* Whether resources removed from the configuration should be stopped (via
	* stop-orphan-resources property)
	*/
	pcmk__sched_stop_removed_resources = (1ULL << 8),

	/*
	* Whether recurring actions removed from the configuration should be
	* cancelled (via stop-orphan-actions property)
	*/
	pcmk__sched_cancel_removed_actions = (1ULL << 9),

	// Whether to stop all resources (via stop-all-resources property)
	pcmk__sched_stop_all = (1ULL << 10),

	// Whether scheduler processing encountered a warning
	pcmk__sched_processing_warning = (1ULL << 11),

	/*
	* Whether start failure should be treated as if
	* \c PCMK_META_MIGRATION_THRESHOLD is 1 (via
	* \c PCMK_OPT_START_FAILURE_IS_FATAL property)
	*/
	pcmk__sched_start_failure_fatal = (1ULL << 12),

	// Unused
	pcmk__sched_remove_after_stop = (1ULL << 13),

	// Whether unseen nodes should be fenced (via startup-fencing property)
	pcmk__sched_startup_fencing = (1ULL << 14),

	/*
	* Whether resources should be left stopped when their node shuts down
	* cleanly (via shutdown-lock property)
	*/
	pcmk__sched_shutdown_lock = (1ULL << 15),

	/*
	* Whether resources' current state should be probed (when unknown) before
	* scheduling any other actions (via the enable-startup-probes property)
	*/
	pcmk__sched_probe_resources = (1ULL << 16),

	// Whether the CIB status section has been parsed yet
	pcmk__sched_have_status = (1ULL << 17),

	// Whether the cluster includes any Pacemaker Remote nodes (via CIB)
	pcmk__sched_have_remote_nodes = (1ULL << 18),


	/* The remaining flags are scheduling options that must be set explicitly */

	/*
	* Whether to skip unpacking the CIB status section and stop the scheduling
	* sequence after applying node-specific location criteria (skipping
	* assignment, ordering, actions, etc.).
	*/
	pcmk__sched_location_only = (1ULL << 20),

	// Whether sensitive resource attributes have been masked
	pcmk__sched_sanitized = (1ULL << 21),

	// Skip counting of total, disabled, and blocked resource instances
	pcmk__sched_no_counts = (1ULL << 23),

	// Whether node scores should be output instead of logged
	pcmk__sched_output_scores = (1ULL << 25),

	// Whether to show node and resource utilization (in log or output)
	pcmk__sched_show_utilization = (1ULL << 26),

	/*
	* Whether to stop the scheduling sequence after unpacking the CIB,
	* calculating cluster status, and applying node health (skipping
	* applying node-specific location criteria, assignment, etc.)
	*/
	pcmk__sched_validate_only = (1ULL << 27),
	};

	// Implementation of pcmk__scheduler_private_t
	struct pcmk__scheduler_private {
	// Be careful about when each piece of information is available and final

	crm_time_t *now; // Time to use when evaluating rules
	pcmk__output_t *out; // Output object for displaying messages
	GHashTable *options; // Cluster options
	const char *fence_action; // Default fencing action
	int fence_timeout_ms; // Value of stonith-timeout property in ms
	const char *placement_strategy; // Value of placement-strategy property
	xmlNode *rsc_defaults; // Configured resource defaults
	xmlNode *op_defaults; // Configured operation defaults
	GList *resources; // Resources in cluster
	GList *actions; // All scheduled actions
	GHashTable *singletons; // Scheduled non-resource actions
	+ int next_action_id; // Counter used as ID for actions
	xmlNode *failed; // History entries of failed actions
	GList *location_constraints; // Location constraints
	GList *colocation_constraints; // Colocation constraints
	GList *ordering_constraints; // Ordering constraints
	GHashTable *ticket_constraints; // Key = ticket ID, value = pcmk__ticket_t
	int next_ordering_id; // Counter used as ID for orderings
	int synapse_count; // Number of transition graph synapses
	};

	// Group of enum pcmk__warnings flags for warnings we want to log once
	extern uint32_t pcmk__warnings;

	/*!
	* \internal
	* \brief Log a resource-tagged message at info severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_info(rsc, fmt, args...) \
	crm_log_tag(LOG_INFO, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log a resource-tagged message at debug severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_debug(rsc, fmt, args...) \
	crm_log_tag(LOG_DEBUG, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log a resource-tagged message at trace severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_trace(rsc, fmt, args...) \
	crm_log_tag(LOG_TRACE, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log an error and remember that current scheduler input has errors
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__sched_err(scheduler, fmt...) do { \
	pcmk__set_scheduler_flags((scheduler), \
	pcmk__sched_processing_error); \
	crm_err(fmt); \
	} while (0)

	/*!
	* \internal
	* \brief Log a warning and remember that current scheduler input has warnings
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__sched_warn(scheduler, fmt...) do { \
	pcmk__set_scheduler_flags((scheduler), \
	pcmk__sched_processing_warning); \
	crm_warn(fmt); \
	} while (0)

	/*!
	* \internal
	* \brief Set scheduler flags
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] flags_to_set Group of enum pcmk__scheduler_flags to set
	*/
	#define pcmk__set_scheduler_flags(scheduler, flags_to_set) do { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", crm_system_name, \
	(scheduler)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	/*!
	* \internal
	* \brief Clear scheduler flags
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] flags_to_clear Group of enum pcmk__scheduler_flags to clear
	*/
	#define pcmk__clear_scheduler_flags(scheduler, flags_to_clear) do { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", crm_system_name, \
	(scheduler)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
	diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
	index 3dc312961a..ffede32120 100644
	--- a/lib/pengine/pe_actions.c
	+++ b/lib/pengine/pe_actions.c
	@@ -1,1799 +1,1799 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml_internal.h>
	#include "pe_status_private.h"

	static void unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms);

	static void
	add_singleton(pcmk_scheduler_t scheduler, pcmk_action_t action)
	{
	if (scheduler->priv->singletons == NULL) {
	scheduler->priv->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(scheduler->priv->singletons, action->uuid, action);
	}

	static pcmk_action_t *
	lookup_singleton(pcmk_scheduler_t scheduler, const char action_uuid)
	{
	/* @TODO This is the only use of the pcmk_scheduler_t:singletons hash table.
	* Compare the performance of this approach to keeping the
	* pcmk_scheduler_t:actions list sorted by action key and just searching
	* that instead.
	*/
	if (scheduler->priv->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(scheduler->priv->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] scheduler Scheduler data
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pcmk_action_t *
	find_existing_action(const char key, const pcmk_resource_t rsc,
	const pcmk_node_t node, const pcmk_scheduler_t scheduler)
	{
	/* When rsc is NULL, it would be quicker to check
	* scheduler->priv->singletons, but checking all scheduler->priv->actions
	* takes the node into account.
	*/
	GList *actions = (rsc == NULL)? scheduler->priv->actions : rsc->priv->actions;
	GList *matches = find_actions(actions, key, node);
	pcmk_action_t *action = NULL;

	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	/*!
	* \internal
	* \brief Find the XML configuration corresponding to a specific action key
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] key "RSC_ACTION_INTERVAL" of action to find
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	static xmlNode *
	find_exact_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	const char *config_name = NULL;
	const char *interval_spec = NULL;
	guint tmp_ms = 0U;

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if (!include_disabled
	&& (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &tmp_ms);
	if (tmp_ms != interval_ms) {
	continue;
	}

	config_name = crm_element_value(operation, PCMK_XA_NAME);
	if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
	return operation;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Find the XML configuration of a resource action
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] action_name Action name to search for
	* \param[in] interval_ms Action interval (in milliseconds) to search for
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	xmlNode *
	pcmk__find_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	xmlNode *action_config = NULL;

	// Try requested action first
	action_config = find_exact_action_config(rsc, action_name, interval_ms,
	include_disabled);

	// For migrate_to and migrate_from actions, retry with "migrate"
	// @TODO This should be either documented or deprecated
	if ((action_config == NULL)
	&& pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	action_config = find_exact_action_config(rsc, "migrate", 0,
	include_disabled);
	}

	return action_config;
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pcmk_action_t *
	new_action(char key, const char task, pcmk_resource_t *rsc,
	const pcmk_node_t node, bool optional, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *action = pcmk__assert_alloc(1, sizeof(pcmk_action_t));

	action->rsc = rsc;
	action->task = pcmk__str_copy(task);
	action->uuid = key;
	action->scheduler = scheduler;

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pcmk__set_action_flags(action, pcmk__action_on_dc);
	}

	pcmk__set_action_flags(action, pcmk__action_runnable);
	if (optional) {
	pcmk__set_action_flags(action, pcmk__action_optional);
	} else {
	pcmk__clear_action_flags(action, pcmk__action_optional);
	}

	if (rsc == NULL) {
	action->meta = pcmk__strkey_table(free, free);
	} else {
	guint interval_ms = 0;

	parse_op_key(key, NULL, NULL, &interval_ms);
	action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
	true);

	/* If the given key is for one of the many notification pseudo-actions
	* (pre_notify_promote, etc.), the actual action name is "notify"
	*/
	if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
	action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
	0, true);
	}

	unpack_operation(action, action->op_entry, interval_ms);
	}

	pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	- scheduler->action_id, key, task,
	+ scheduler->priv->next_action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	pcmk__node_name(node));
	- action->id = scheduler->action_id++;
	+ action->id = scheduler->priv->next_action_id++;

	scheduler->priv->actions = g_list_prepend(scheduler->priv->actions, action);
	if (rsc == NULL) {
	add_singleton(scheduler, action);
	} else {
	rsc->priv->actions = g_list_prepend(rsc->priv->actions, action);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Unpack a resource's action-specific instance parameters
	*
	* \param[in] action_xml XML of action's configuration in CIB (if any)
	* \param[in,out] node_attrs Table of node attributes (for rule evaluation)
	* \param[in,out] scheduler Cluster working set (for rule evaluation)
	*
	* \return Newly allocated hash table of action-specific instance parameters
	*/
	GHashTable *
	pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = node_attrs,
	.now = scheduler->priv->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__unpack_dataset_nvpairs(action_xml, PCMK_XE_INSTANCE_ATTRIBUTES,
	&rule_data, params, NULL,
	FALSE, scheduler);
	return params;
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in,out] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pcmk_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pcmk__action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pcmk__rsc_managed)
	&& (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is optional (%s is unmanaged)",
	action->uuid, pcmk__node_name(action->node),
	action->rsc->id);
	pcmk__set_action_flags(action, pcmk__action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pcmk__clear_action_flags(action, pcmk__action_optional);
	}
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	enum pe_quorum_policy policy = scheduler->no_quorum_policy;

	if (pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
	policy = pcmk_no_quorum_ignore;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
	switch (rsc->priv->orig_role) {
	case pcmk_role_promoted:
	case pcmk_role_unpromoted:
	if (rsc->priv->next_role > pcmk_role_unpromoted) {
	pe__set_next_role(rsc, pcmk_role_unpromoted,
	PCMK_OPT_NO_QUORUM_POLICY "=demote");
	}
	policy = pcmk_no_quorum_ignore;
	break;
	default:
	policy = pcmk_no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in,out] action Action to update
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pcmk_action_t *action,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = action->rsc;

	if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pcmk__rsc_trace(rsc, "%s is unrunnable (unallocated)", action->uuid);
	pcmk__clear_action_flags(action, pcmk__action_runnable);

	} else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
	&& !(action->node->details->online)
	&& (!pcmk__is_guest_or_bundle_node(action->node)
	\|\| pcmk_is_set(action->node->priv->flags,
	pcmk__node_remote_reset))) {
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
	action->uuid, pcmk__node_name(action->node));
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
	&& action->node->details->pending) {
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	do_crm_log(LOG_WARNING,
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, pcmk__node_name(action->node));

	} else if (action->needs == pcmk__requires_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pcmk__is_guest_or_bundle_node(action->node)
	&& !pe_can_fence(scheduler, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pcmk__rsc_debug(rsc,
	"%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	} else {
	pcmk__rsc_trace(rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, pcmk__node_name(action->node));
	pcmk__set_action_flags(action, pcmk__action_runnable);
	}

	} else {
	switch (effective_quorum_policy(rsc, scheduler)) {
	case pcmk_no_quorum_stop:
	pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid, pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case pcmk_no_quorum_freeze:
	if (!rsc->priv->fns->active(rsc, TRUE)
	\|\| (rsc->priv->next_role > rsc->priv->orig_role)) {
	pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
	action->uuid,
	pcmk__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk__action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pcmk__set_action_flags(action, pcmk__action_runnable);
	break;
	}
	}
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value,
	PCMK_VALUE_STANDBY, PCMK_VALUE_DEMOTE,
	PCMK_VALUE_STOP, NULL);
	}

	/*!
	* \internal
	* \brief Validate (and possibly reset) resource action's on_fail meta-attribute
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] action_config Action configuration XML from CIB (if any)
	* \param[in,out] meta Table of action meta-attributes
	*/
	static void
	validate_on_fail(const pcmk_resource_t rsc, const char action_name,
	const xmlNode action_config, GHashTable meta)
	{
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);
	guint interval_ms = 0U;

	// Stop actions can only use certain on-fail values
	if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", rsc->id, value);
	g_hash_table_remove(meta, PCMK_META_ON_FAIL);
	return;
	}

	/* Demote actions default on-fail to the on-fail value for the first
	* recurring monitor for the promoted role (if any).
	*/
	if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
	&& (value == NULL)) {

	/* @TODO This does not consider promote options set in a meta-attribute
	* block (which may have rules that need to be evaluated) rather than
	* XML properties.
	*/
	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	const char *promote_on_fail = NULL;

	/* We only care about explicit on-fail (if promote uses default, so
	* can demote)
	*/
	promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL);
	if (promote_on_fail == NULL) {
	continue;
	}

	// We only care about recurring monitors for the promoted role
	name = crm_element_value(operation, PCMK_XA_NAME);
	role = crm_element_value(operation, PCMK_XA_ROLE);
	if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
	continue;
	}
	interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// We only care about enabled monitors
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	/* Demote actions can't default to
	* PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
	*/
	if (pcmk__str_eq(promote_on_fail, PCMK_VALUE_DEMOTE,
	pcmk__str_casei)) {
	continue;
	}

	// Use value from first applicable promote action found
	pcmk__insert_dup(meta, PCMK_META_ON_FAIL, promote_on_fail);
	}
	return;
	}

	if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
	&& !pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {

	pcmk__insert_dup(meta, PCMK_META_ON_FAIL, PCMK_VALUE_IGNORE);
	return;
	}

	// PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE is allowed only for certain actions
	if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
	name = crm_element_value(action_config, PCMK_XA_NAME);
	role = crm_element_value(action_config, PCMK_XA_ROLE);
	interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);

	if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
	&& ((interval_ms == 0U)
	\|\| !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL))) {

	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", rsc->id, name);
	g_hash_table_remove(meta, PCMK_META_ON_FAIL);
	return;
	}
	}
	}

	static int
	unpack_timeout(const char *value)
	{
	long long timeout_ms = crm_get_msec(value);

	if (timeout_ms <= 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}
	return (int) QB_MIN(timeout_ms, INT_MAX);
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, const xmlNode xml_obj,
	guint interval_ms, const crm_time_t *now,
	long long *start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for "
	"operation '%s' because '%s' is not valid",
	pcmk__s(pcmk__xe_id(xml_obj), "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result, pcmk__s(pcmk__xe_id(xml_obj), "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	long long start_delay_ms = 0;

	if (value == NULL) {
	return 0;
	}

	start_delay_ms = crm_get_msec(value);
	start_delay_ms = QB_MIN(start_delay_ms, INT_MAX);
	if (start_delay_ms < 0) {
	start_delay_ms = 0;
	}

	if (meta != NULL) {
	g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY),
	pcmk__itoa(start_delay_ms));
	}

	return (int) start_delay_ms;
	}

	/*!
	* \internal
	* \brief Find a resource's most frequent recurring monitor
	*
	* \param[in] rsc Resource to check
	*
	* \return Operation XML configured for most frequent recurring monitor for
	* \p rsc (if any)
	*/
	static xmlNode *
	most_frequent_monitor(const pcmk_resource_t *rsc)
	{
	guint min_interval_ms = G_MAXUINT;
	xmlNode *op = NULL;

	for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
	PCMK_XE_OP, NULL, NULL);
	operation != NULL; operation = pcmk__xe_next_same(operation)) {

	bool enabled = false;
	guint interval_ms = 0U;
	const char *interval_spec = crm_element_value(operation,
	PCMK_META_INTERVAL);

	// We only care about enabled recurring monitors
	if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
	PCMK_ACTION_MONITOR, pcmk__str_none)) {
	continue;
	}

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	if (interval_ms < min_interval_ms) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	return op;
	}

	/*!
	* \internal
	* \brief Unpack action meta-attributes
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node that action is on
	* \param[in] action_name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] action_config Action XML configuration from CIB (if any)
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds) from its CIB XML
	* configuration (including defaults).
	*
	* \return Newly allocated hash table with normalized action meta-attributes
	*/
	GHashTable *
	pcmk__unpack_action_meta(pcmk_resource_t rsc, const pcmk_node_t node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config)
	{
	GHashTable *meta = NULL;
	const char *timeout_spec = NULL;
	const char *str = NULL;

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
	.provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
	.agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE),
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action_name,
	.interval = interval_ms,
	};

	pe_rule_eval_data_t rule_data = {
	/* @COMPAT Support for node attribute expressions in operation
	* meta-attributes (whether in the operation configuration or operation
	* defaults) is deprecated. When we can break behavioral backward
	* compatibility, drop this line.
	*/
	.node_hash = (node == NULL)? NULL : node->priv->attrs,

	.now = rsc->priv->scheduler->priv->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data,
	};

	meta = pcmk__strkey_table(free, free);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(rsc->priv->scheduler->priv->op_defaults,
	PCMK_XE_META_ATTRIBUTES, &rule_data, meta, NULL,
	FALSE, rsc->priv->scheduler);

	// Derive default timeout for probes from recurring monitor timeouts
	if (pcmk_is_probe(action_name, interval_ms)) {
	xmlNode *min_interval_mon = most_frequent_monitor(rsc);

	if (min_interval_mon != NULL) {
	/* @TODO This does not consider timeouts set in
	* PCMK_XE_META_ATTRIBUTES blocks (which may also have rules that
	* need to be evaluated).
	*/
	timeout_spec = crm_element_value(min_interval_mon,
	PCMK_META_TIMEOUT);
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting default timeout for %s probe to "
	"most frequent monitor's timeout '%s'",
	rsc->id, timeout_spec);
	pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
	}
	}
	}

	if (action_config != NULL) {
	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(action_config, PCMK_XE_META_ATTRIBUTES,
	&rule_data, meta, NULL, TRUE,
	rsc->priv->scheduler);

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	* (See below for the only exception, fence device start/probe timeout.)
	*/
	for (xmlAttrPtr attr = action_config->properties;
	attr != NULL; attr = attr->next) {
	pcmk__insert_dup(meta, (const char *) attr->name,
	pcmk__xml_attr_value(attr));
	}
	}

	g_hash_table_remove(meta, PCMK_XA_ID);

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_INTERVAL),
	crm_strdup_printf("%u", interval_ms));
	} else {
	g_hash_table_remove(meta, PCMK_META_INTERVAL);
	}

	/* Timeout order of precedence (highest to lowest):
	* 1. pcmk_monitor_timeout resource parameter (only for starts and probes
	* when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
	* monitors via the executor instead)
	* 2. timeout configured in <op> (with <op timeout> taking precedence over
	* <op> <meta_attributes>)
	* 3. timeout configured in <op_defaults> <meta_attributes>
	* 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
	*/

	// Check for pcmk_monitor_timeout
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
	\|\| pcmk_is_probe(action_name, interval_ms))) {

	GHashTable *params = pe_rsc_params(rsc, node, rsc->priv->scheduler);

	timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting timeout for %s %s to "
	"pcmk_monitor_timeout (%s)",
	rsc->id, action_name, timeout_spec);
	pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
	}
	}

	// Normalize timeout to positive milliseconds
	timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT);
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_TIMEOUT),
	pcmk__itoa(unpack_timeout(timeout_spec)));

	// Ensure on-fail has a valid value
	validate_on_fail(rsc, action_name, action_config, meta);

	// Normalize PCMK_META_START_DELAY
	str = g_hash_table_lookup(meta, PCMK_META_START_DELAY);
	if (str != NULL) {
	unpack_start_delay(str, meta);
	} else {
	long long start_delay = 0;

	str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN);
	if (unpack_interval_origin(str, action_config, interval_ms,
	rsc->priv->scheduler->priv->now,
	&start_delay)) {
	g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_START_DELAY),
	crm_strdup_printf("%lld", start_delay));
	}
	}
	return meta;
	}

	/*!
	* \internal
	* \brief Determine an action's quorum and fencing dependency
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action being unpacked
	*
	* \return Quorum and fencing dependency appropriate to action
	*/
	enum pcmk__requires
	pcmk__action_requires(const pcmk_resource_t rsc, const char action_name)
	{
	const char *value = NULL;
	enum pcmk__requires requires = pcmk__requires_nothing;

	CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);

	if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
	requires = pcmk__requires_fencing;
	value = "fencing";

	} else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_quorum)) {
	requires = pcmk__requires_quorum;
	value = "quorum";

	} else {
	value = "nothing";
	}
	pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
	return requires;
	}

	/*!
	* \internal
	* \brief Parse action failure response from a user-provided string
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] value User-provided configuration value for on-fail
	*
	* \return Action failure response parsed from \p text
	*/
	enum pcmk__on_fail
	pcmk__parse_on_fail(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, const char *value)
	{
	const char *desc = NULL;
	bool needs_remote_reset = false;
	enum pcmk__on_fail on_fail = pcmk__on_fail_ignore;
	const pcmk_scheduler_t *scheduler = NULL;

	// There's no enum value for unknown or invalid, so assert
	CRM_ASSERT((rsc != NULL) && (action_name != NULL));
	scheduler = rsc->priv->scheduler;

	if (value == NULL) {
	// Use default

	} else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_block;
	desc = "block";

	} else if (pcmk__str_eq(value, PCMK_VALUE_FENCE, pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	on_fail = pcmk__on_fail_fence_node;
	desc = "node fencing";
	} else {
	pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for "
	"%s of %s to 'stop' because 'fence' is not "
	"valid when fencing is disabled",
	action_name, rsc->id);
	on_fail = pcmk__on_fail_stop;
	desc = "stop resource";
	}

	} else if (pcmk__str_eq(value, PCMK_VALUE_STANDBY, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_standby_node;
	desc = "node standby";

	} else if (pcmk__strcase_any_of(value,
	PCMK_VALUE_IGNORE, PCMK_VALUE_NOTHING,
	NULL)) {
	desc = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	on_fail = pcmk__on_fail_ban;
	desc = "force migration";

	} else if (pcmk__str_eq(value, PCMK_VALUE_STOP, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_stop;
	desc = "stop resource";

	} else if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_restart;
	desc = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, PCMK_VALUE_RESTART_CONTAINER,
	pcmk__str_casei)) {
	if (rsc->priv->launcher == NULL) {
	pcmk__rsc_debug(rsc,
	"Using default " PCMK_META_ON_FAIL " for %s "
	"of %s because it does not have a launcher",
	action_name, rsc->id);
	} else {
	on_fail = pcmk__on_fail_restart_container;
	desc = "restart container (and possibly migrate)";
	}

	} else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
	on_fail = pcmk__on_fail_demote;
	desc = "demote instance";

	} else {
	pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for "
	"%s of %s because '%s' is not valid",
	action_name, rsc->id, value);
	}

	/* Remote node connections are handled specially. Failures that result
	* in dropping an active connection must result in fencing. The only
	* failures that don't are probes and starts. The user can explicitly set
	* PCMK_META_ON_FAIL=PCMK_VALUE_FENCE to fence after start failures.
	*/
	if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
	&& pcmk__is_remote_node(pcmk_find_node(scheduler, rsc->id))
	&& !pcmk_is_probe(action_name, interval_ms)
	&& !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
	needs_remote_reset = true;
	if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	desc = NULL; // Force default for unmanaged connections
	}
	}

	if (desc != NULL) {
	// Explicit value used, default not needed

	} else if (rsc->priv->launcher != NULL) {
	on_fail = pcmk__on_fail_restart_container;
	desc = "restart container (and possibly migrate) (default)";

	} else if (needs_remote_reset) {
	if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	desc = "fence remote node (default)";
	} else {
	desc = "recover remote node connection (default)";
	}
	on_fail = pcmk__on_fail_reset_remote;
	} else {
	on_fail = pcmk__on_fail_stop;
	desc = "stop unmanaged remote node (enforcing default)";
	}

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
	if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
	on_fail = pcmk__on_fail_fence_node;
	desc = "resource fence (default)";
	} else {
	on_fail = pcmk__on_fail_block;
	desc = "resource block (default)";
	}

	} else {
	on_fail = pcmk__on_fail_restart;
	desc = "restart (and possibly migrate) (default)";
	}

	pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
	pcmk__readable_interval(interval_ms), action_name,
	rsc->id, desc);
	return on_fail;
	}

	/*!
	* \internal
	* \brief Determine a resource's role after failure of an action
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] on_fail Failure handling for action
	* \param[in] meta Unpacked action meta-attributes
	*
	* \return Resource role that results from failure of action
	*/
	enum rsc_role_e
	pcmk__role_after_failure(const pcmk_resource_t rsc, const char action_name,
	enum pcmk__on_fail on_fail, GHashTable *meta)
	{
	const char *value = NULL;
	enum rsc_role_e role = pcmk_role_unknown;

	// Set default for role after failure specially in certain circumstances
	switch (on_fail) {
	case pcmk__on_fail_stop:
	role = pcmk_role_stopped;
	break;

	case pcmk__on_fail_reset_remote:
	if (rsc->priv->remote_reconnect_ms != 0U) {
	role = pcmk_role_stopped;
	}
	break;

	default:
	break;
	}

	// @COMPAT Check for explicitly configured role (deprecated)
	value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE);
	if (value != NULL) {
	pcmk__warn_once(pcmk__wo_role_after,
	"Support for " PCMK__META_ROLE_AFTER_FAILURE " is "
	"deprecated and will be removed in a future release");
	if (role == pcmk_role_unknown) {
	role = pcmk_parse_role(value);
	if (role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring invalid value %s for "
	PCMK__META_ROLE_AFTER_FAILURE,
	value);
	}
	}
	}

	if (role == pcmk_role_unknown) {
	// Use default
	if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	role = pcmk_role_unpromoted;
	} else {
	role = pcmk_role_started;
	}
	}
	pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
	rsc->id, action_name, pcmk_role_text(role));
	return role;
	}

	/*!
	* \internal
	* \brief Unpack action configuration
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds), requirements, and
	* failure policy from its CIB XML configuration (including defaults).
	*
	* \param[in,out] action Resource action to unpack into
	* \param[in] xml_obj Action configuration XML (NULL for defaults only)
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms)
	{
	const char *value = NULL;

	action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
	action->task, interval_ms, xml_obj);
	action->needs = pcmk__action_requires(action->rsc, action->task);

	value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL);
	action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
	interval_ms, value);

	action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
	action->on_fail, action->meta);
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in,out] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Action object corresponding to arguments (guaranteed not to be
	* \c NULL)
	* \note This function takes ownership of (and might free) \p key, and
	* \p scheduler takes ownership of the returned action (the caller should
	* not free it).
	*/
	pcmk_action_t *
	custom_action(pcmk_resource_t rsc, char key, const char *task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL));

	action = find_existing_action(key, rsc, on_node, scheduler);
	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, scheduler);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	/* An action can be initially created with a NULL node, and later have
	* the node added via find_existing_action() (above) -> find_actions().
	* That is why the extra parameters are unpacked here rather than in
	* new_action().
	*/
	if ((action->node != NULL) && (action->op_entry != NULL)
	&& !pcmk_is_set(action->flags, pcmk__action_attrs_evaluated)) {

	GHashTable *attrs = action->node->priv->attrs;

	if (action->extra != NULL) {
	g_hash_table_destroy(action->extra);
	}
	action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
	attrs, scheduler);
	pcmk__set_action_flags(action, pcmk__action_attrs_evaluated);
	}

	update_resource_action_runnable(action, scheduler);
	}

	if (action->extra == NULL) {
	action->extra = pcmk__strkey_table(free, free);
	}

	return action;
	}

	pcmk_action_t *
	get_pseudo_op(const char name, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *op = lookup_singleton(scheduler, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
	pcmk__set_action_flags(op, pcmk__action_pseudo\|pcmk__action_runnable);
	}
	return op;
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *candidate = gIter->data;

	if (candidate->priv->children != NULL) {
	matches = find_unfencing_devices(candidate->priv->children,
	matches);

	} else if (!pcmk_is_set(candidate->flags, pcmk__rsc_fence_device)) {
	continue;

	} else if (pcmk_is_set(candidate->flags, pcmk__rsc_needs_unfencing)) {
	matches = g_list_prepend(matches, candidate);

	} else if (pcmk__str_eq(g_hash_table_lookup(candidate->priv->meta,
	PCMK_STONITH_PROVIDES),
	PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(const pcmk_node_t *node,
	const pcmk_scheduler_t *scheduler)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
	if (scheduler->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->priv->variant != pcmk__node_variant_cluster) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *n = gIter->data;

	if (n->priv->variant != pcmk__node_variant_cluster) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->priv->priority > top_priority) {
	top_priority = n->priv->priority;
	}

	if (member_count == 1
	\|\| n->priv->priority < lowest_priority) {
	lowest_priority = n->priv->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->priv->priority < top_priority) {
	return 0;
	}

	return scheduler->priority_fencing_delay;
	}

	pcmk_action_t *
	pe_fence_op(pcmk_node_t node, const char op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler)
	{
	char *op_key = NULL;
	pcmk_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = scheduler->priv->fence_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s",
	PCMK_ACTION_STONITH, node->priv->name, op);

	stonith_op = lookup_singleton(scheduler, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
	TRUE, scheduler);

	pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE, node->priv->name);
	pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE_UUID,
	node->priv->id);
	pcmk__insert_meta(stonith_op, PCMK__META_STONITH_ACTION, op);

	if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
	/* Extra work to detect device changes
	*/
	GString *digests_all = g_string_sized_new(1024);
	GString *digests_secure = g_string_sized_new(1024);

	GList *matches = find_unfencing_devices(scheduler->priv->resources,
	NULL);

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->priv->meta,
	PCMK_XA_TYPE);
	pcmk__op_digest_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node,
	scheduler);
	if (data->rc == pcmk__digest_mismatch) {
	optional = FALSE;
	crm_notice("Unfencing node %s because the definition of "
	"%s changed", pcmk__node_name(node), match->id);
	if (!pcmk__is_daemon && (scheduler->priv->out != NULL)) {
	pcmk__output_t *out = scheduler->priv->out;

	out->info(out,
	"notice: Unfencing node %s because the "
	"definition of %s changed",
	pcmk__node_name(node), match->id);
	}
	}

	pcmk__g_strcat(digests_all,
	match->id, ":", agent, ":",
	data->digest_all_calc, ",", NULL);
	pcmk__g_strcat(digests_secure,
	match->id, ":", agent, ":",
	data->digest_secure_calc, ",", NULL);
	}
	pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_ALL,
	digests_all->str);
	g_string_free(digests_all, TRUE);

	pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_SECURE,
	digests_secure->str);
	g_string_free(digests_secure, TRUE);
	}

	} else {
	free(op_key);
	}

	if (scheduler->priority_fencing_delay > 0

	/* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
	* applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
	* an indicator.
	*/
	&& (priority_delay

	/* The priority delay needs to be recalculated if this function has
	* been called by schedule_fencing_and_shutdowns() after node
	* priority has already been calculated by native_add_running().
	*/
	\|\| g_hash_table_lookup(stonith_op->meta,
	PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
	* it's 0 for the targeting node. So that it takes precedence over
	* any possible `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
	scheduler));

	g_hash_table_insert(stonith_op->meta,
	strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(scheduler, node)) {
	pcmk__clear_action_flags(stonith_op, pcmk__action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	pe_free_action(pcmk_action_t *action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free);
	g_list_free_full(action->actions_after, free);
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	enum pcmk__action_type
	get_complex_task(const pcmk_resource_t rsc, const char name)
	{
	enum pcmk__action_type task = pcmk__parse_action(name);

	if (pcmk__is_primitive(rsc)) {
	switch (task) {
	case pcmk__action_stopped:
	case pcmk__action_started:
	case pcmk__action_demoted:
	case pcmk__action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s",
	name, rsc->id);
	--task;
	break;
	default:
	break;
	}
	}
	return task;
	}

	/*!
	* \internal
	* \brief Find first matching action in a list
	*
	* \param[in] input List of actions to search
	* \param[in] uuid If not NULL, action must have this UUID
	* \param[in] task If not NULL, action must have this action name
	* \param[in] on_node If not NULL, action must be on this node
	*
	* \return First action in list that matches criteria, or NULL if none
	*/
	pcmk_action_t *
	find_first_action(const GList input, const char uuid, const char *task,
	const pcmk_node_t *on_node)
	{
	CRM_CHECK(uuid \|\| task, return NULL);

	for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (pcmk__same_node(on_node, action->node)) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, pcmk__node_name(on_node));

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (pcmk__same_node(on_node, action->node)) {
	crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__same_node(on_node, action->node)) {

	crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->priv->actions, key, node);
	} else {
	result = find_actions(rsc->priv->actions, key, node);
	}
	free(key);
	return result;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(const pcmk_action_t *action, enum pcmk__action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pcmk__action_runnable:
	change = "unrunnable";
	break;
	case pcmk__action_migratable:
	change = "unmigrateable";
	break;
	case pcmk__action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason,
	pcmk__s(reason, "(none)"));
	} else if (action->reason == NULL) {
	pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, pcmk__s(reason, "(none)"));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in,out] rsc Resource to clear
	* \param[in] node Node to clear history on
	*/
	void
	pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	CRM_ASSERT((rsc != NULL) && (node != NULL));

	custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->priv->scheduler);
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	int
	pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
	const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);

	const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE);
	const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE);
	bool same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);

	if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
	/* We have duplicate PCMK__XE_LRM_RSC_OP entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pcmk__config_err("Duplicate " PCMK__XE_LRM_RSC_OP " entries named %s",
	a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, PCMK__XA_CALL_ID, &a_call_id);
	crm_element_value_int(xml_b, PCMK__XA_CALL_ID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (a_call_id >= 0 && b_call_id >= 0
	&& (!same_node \|\| a_call_id == b_call_id)) {
	/* The op and last_failed_op are the same. Order on
	* PCMK_XA_LAST_RC_CHANGE.
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, PCMK_XA_LAST_RC_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, PCMK_XA_LAST_RC_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation.
	* Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative
	* to the other.
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a,
	PCMK__XA_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b,
	PCMK__XA_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));
	}

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	return pe__is_newer_op((const xmlNode ) a, (const xmlNode ) b);
	}

	/*!
	* \internal
	* \brief Create a new pseudo-action for a resource
	*
	* \param[in,out] rsc Resource to create action for
	* \param[in] task Action name
	* \param[in] optional Whether action should be considered optional
	* \param[in] runnable Whethe action should be considered runnable
	*
	* \return New action object corresponding to arguments
	*/
	pcmk_action_t *
	pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char task, bool optional,
	bool runnable)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL));

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
	optional, rsc->priv->scheduler);
	pcmk__set_action_flags(action, pcmk__action_pseudo);
	if (runnable) {
	pcmk__set_action_flags(action, pcmk__action_runnable);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Add the expected result to an action
	*
	* \param[in,out] action Action to add expected result to
	* \param[in] expected_result Expected result to add
	*
	* \note This is more efficient than calling pcmk__insert_meta().
	*/
	void
	pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
	{
	CRM_ASSERT((action != NULL) && (action->meta != NULL));

	g_hash_table_insert(action->meta, pcmk__str_copy(PCMK__META_OP_TARGET_RC),
	pcmk__itoa(expected_result));
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index 2be388b15c..1d8f093808 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,528 +1,528 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/cib_internal.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler == NULL) {
	return NULL;
	}
	scheduler->priv = calloc(1, sizeof(pcmk__scheduler_private_t));
	if (scheduler->priv == NULL) {
	free(scheduler);
	return NULL;
	}
	set_working_set_defaults(scheduler);
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	free(scheduler->priv);
	free(scheduler);
	}
	}

	#define XPATH_DEPRECATED_RULES \
	"//" PCMK_XE_OP_DEFAULTS "//" PCMK_XE_EXPRESSION \
	"\|//" PCMK_XE_OP "//" PCMK_XE_EXPRESSION

	/*!
	* \internal
	* \brief Log a warning for deprecated rule syntax in operations
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	check_for_deprecated_rules(pcmk_scheduler_t *scheduler)
	{
	// @COMPAT Drop this function when support for the syntax is dropped
	xmlNode *deprecated = get_xpath_object(XPATH_DEPRECATED_RULES,
	scheduler->input, LOG_NEVER);

	if (deprecated != NULL) {
	pcmk__warn_once(pcmk__wo_op_attr_expr,
	"Support for rules with node attribute expressions in "
	PCMK_XE_OP " or " PCMK_XE_OP_DEFAULTS " is deprecated "
	"and will be dropped in a future release");
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	const char *new_version = NULL;
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	new_version = crm_element_value(scheduler->input, PCMK_XA_CRM_FEATURE_SET);

	if (pcmk__check_feature_set(new_version) != pcmk_rc_ok) {
	pcmk__config_err("Can't process CIB with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->priv->failed != NULL) {
	pcmk__xml_free(scheduler->priv->failed);
	}
	scheduler->priv->failed = pcmk__xe_create(NULL, "failed-ops");

	if (scheduler->priv->now == NULL) {
	scheduler->priv->now = crm_time_new(NULL);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, PCMK_XA_HAVE_QUORUM)) {
	pcmk__set_scheduler_flags(scheduler, pcmk__sched_quorate);
	} else {
	pcmk__clear_scheduler_flags(scheduler, pcmk__sched_quorate);
	}

	scheduler->priv->op_defaults = get_xpath_object("//" PCMK_XE_OP_DEFAULTS,
	scheduler->input,
	LOG_NEVER);
	check_for_deprecated_rules(scheduler);

	scheduler->priv->rsc_defaults = get_xpath_object("//" PCMK_XE_RSC_DEFAULTS,
	scheduler->input,
	LOG_NEVER);

	section = get_xpath_object("//" PCMK_XE_CRM_CONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk__sched_location_only\|pcmk__sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	pcmk__sched_warn(scheduler,
	"Fencing and resource management disabled "
	"due to lack of quorum");
	}

	section = get_xpath_object("//" PCMK_XE_NODES, scheduler->input, LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" PCMK_XE_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" PCMK_XE_FENCING_TOPOLOGY, scheduler->input,
	LOG_TRACE);
	pcmk__unpack_fencing_topology(section, scheduler);

	section = get_xpath_object("//" PCMK_XE_TAGS, scheduler->input, LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
	section = get_xpath_object("//" PCMK_XE_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk__sched_no_counts)) {
	for (GList *item = scheduler->priv->resources;
	item != NULL; item = item->next) {

	pcmk_resource_t *rsc = item->data;

	rsc->priv->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->priv->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->private->name for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pcmk__is_pacemaker_remote_node(node)?
	"(guest or remote)" : pcmk__node_name(node)));

	if (node->priv->attrs != NULL) {
	g_hash_table_destroy(node->priv->attrs);
	}
	if (node->priv->utilization != NULL) {
	g_hash_table_destroy(node->priv->utilization);
	}
	if (node->priv->digest_cache != NULL) {
	g_hash_table_destroy(node->priv->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->priv->assigned_resources);
	free(node->priv);
	free(node->details);
	free(node->assign);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__action_relation_t *order = iterator->data;

	iterator = iterator->next;

	free(order->task1);
	free(order->task2);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	pcmk__clear_scheduler_flags(scheduler, pcmk__sched_have_status);
	if (scheduler->priv->options != NULL) {
	g_hash_table_destroy(scheduler->priv->options);
	}

	if (scheduler->priv->singletons != NULL) {
	g_hash_table_destroy(scheduler->priv->singletons);
	}

	if (scheduler->priv->ticket_constraints != NULL) {
	g_hash_table_destroy(scheduler->priv->ticket_constraints);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	crm_trace("deleting resources");
	pe_free_resources(scheduler->priv->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->priv->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	pcmk__xml_free(scheduler->graph);
	crm_time_free(scheduler->priv->now);
	pcmk__xml_free(scheduler->input);
	pcmk__xml_free(scheduler->priv->failed);

	set_working_set_defaults(scheduler);

	CRM_LOG_ASSERT((scheduler->priv->location_constraints == NULL)
	&& (scheduler->priv->ordering_constraints == NULL));
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->priv->ordering_constraints));
	pe__free_ordering(scheduler->priv->ordering_constraints);
	scheduler->priv->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->priv->location_constraints));
	pe__free_location(scheduler->priv->location_constraints);
	scheduler->priv->location_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->priv->colocation_constraints));
	g_list_free_full(scheduler->priv->colocation_constraints, free);
	scheduler->priv->colocation_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	// These members must be preserved
	pcmk__scheduler_private_t *priv = scheduler->priv;
	pcmk__output_t *out = priv->out;

	// Wipe the main structs (any other members must have previously been freed)
	memset(scheduler, 0, sizeof(pcmk_scheduler_t));
	memset(priv, 0, sizeof(pcmk__scheduler_private_t));

	// Restore the members to preserve
	scheduler->priv = priv;
	scheduler->priv->out = out;

	// Set defaults for everything else
	scheduler->priv->next_ordering_id = 1;
	- scheduler->action_id = 1;
	+ scheduler->priv->next_action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	pcmk__set_scheduler_flags(scheduler,
	pcmk__sched_symmetric_cluster
	\|pcmk__sched_stop_removed_resources
	\|pcmk__sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, PCMK_VALUE_TRUE)) {
	pcmk__set_scheduler_flags(scheduler, pcmk__sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;
	pcmk_resource_t *match = parent->priv->fns->find_rsc(parent, id, NULL,
	flags);

	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pcmk__find_node_in_list(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->priv->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/status_compat.h>

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	return pcmk__find_node_in_list(nodes, node_name);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pengine/tests/status/set_working_set_defaults_test.c b/lib/pengine/tests/status/set_working_set_defaults_test.c
	index 733573911e..2770a16213 100644
	--- a/lib/pengine/tests/status/set_working_set_defaults_test.c
	+++ b/lib/pengine/tests/status/set_working_set_defaults_test.c
	@@ -1,52 +1,52 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>

	#include <crm/common/scheduler.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/status.h>

	#include "mock_private.h"

	static void
	check_defaults(void **state) {
	uint32_t flags;
	pcmk_scheduler_t *scheduler = pcmk__assert_alloc(1,
	sizeof(pcmk_scheduler_t));

	scheduler->priv = pcmk__assert_alloc(1, sizeof(pcmk__scheduler_private_t));
	set_working_set_defaults(scheduler);

	flags = pcmk__sched_symmetric_cluster
	\|pcmk__sched_stop_removed_resources
	\|pcmk__sched_cancel_removed_actions;

	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, PCMK_VALUE_TRUE)) {
	flags \|= pcmk__sched_concurrent_fencing;
	}


	assert_null(scheduler->priv->out);
	assert_int_equal(scheduler->priv->next_ordering_id, 1);
	- assert_int_equal(scheduler->action_id, 1);
	+ assert_int_equal(scheduler->priv->next_action_id, 1);
	assert_int_equal(scheduler->no_quorum_policy, pcmk_no_quorum_stop);
	assert_int_equal(scheduler->flags, flags);

	/* Avoid calling pe_free_working_set here so we don't artificially
	* inflate the coverage numbers.
	*/
	free(scheduler->priv);
	free(scheduler);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(check_defaults))

File Metadata

Mime Type: text/x-diff
Expires: Fri, Sep 5, 9:20 AM (9 h, 9 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2301539
Default Alt Text: (99 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions