No OneTemporary
Actions

Size

690 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index 135ae19afb..76e8c4eb4a 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,648 +1,638 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H

	# include <stdbool.h>
	# include <stdint.h>
	# include <string.h>
	# include <crm/msg_xml.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>
	# include <crm/common/scheduler_internal.h>

	const char pe__resource_description(const pcmk_resource_t rsc,
	uint32_t show_opts);

	bool pe__clone_is_ordered(const pcmk_resource_t *clone);
	int pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag);
	bool pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags);

	bool pe__group_flag_is_set(const pcmk_resource_t *group, uint32_t flags);
	pcmk_resource_t pe__last_group_member(const pcmk_resource_t group);

	-# define pe_err(fmt...) do { \
	- was_processing_error = TRUE; \
	- pcmk__config_err(fmt); \
	- } while (0)
	-
	-# define pe_warn(fmt...) do { \
	- was_processing_warning = TRUE; \
	- pcmk__config_warn(fmt); \
	- } while (0)
	-
	#define pe__set_working_set_flags(scheduler, flags_to_set) do { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", crm_system_name, \
	(scheduler)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(scheduler, flags_to_clear) do { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", crm_system_name, \
	(scheduler)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pcmk__warnings, pe_wo_bit)) { \
	if (pe_wo_bit == pcmk__wo_blind) { \
	crm_warn(fmt); \
	} else { \
	- pe_warn(fmt); \
	+ pcmk__config_warn(fmt); \
	} \
	pcmk__warnings = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Warn-once", "logging", \
	pcmk__warnings, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);

	const pcmk_resource_t pe__const_top_resource(const pcmk_resource_t rsc,
	bool include_bundle);

	int pe__clone_max(const pcmk_resource_t *clone);
	int pe__clone_node_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_node_max(const pcmk_resource_t *clone);
	void pe__create_clone_notifications(pcmk_resource_t *clone);
	void pe__free_clone_notification_data(pcmk_resource_t *clone);
	void pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
	pcmk_action_t *start,
	pcmk_action_t *started,
	pcmk_action_t *stop,
	pcmk_action_t *stopped);

	pcmk_action_t pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char *task,
	bool optional, bool runnable);

	void pe__create_promotable_pseudo_ops(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting);

	bool pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node);

	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pcmk_resource_t rsc, pcmk_node_t *node, gboolean create,
	const char name, pcmk_scheduler_t scheduler);
	pcmk_node_t native_location(const pcmk_resource_t rsc, GList **list,
	int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed);

	gboolean native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean group_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean clone_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);

	pcmk_resource_t native_find_rsc(pcmk_resource_t rsc, const char *id,
	const pcmk_node_t *node, int flags);

	gboolean native_active(pcmk_resource_t *rsc, gboolean all);
	gboolean group_active(pcmk_resource_t *rsc, gboolean all);
	gboolean clone_active(pcmk_resource_t *rsc, gboolean all);
	gboolean pe__bundle_active(pcmk_resource_t *rsc, gboolean all);

	//! \deprecated This function will be removed in a future release
	void native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void group_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void clone_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar pcmk__native_output_string(const pcmk_resource_t rsc, const char *name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pcmk_node_t node, bool print_detail);


	// Clone notifications (pe_notif.c)
	void pe__order_notifs_after_fencing(const pcmk_action_t *action,
	pcmk_resource_t *rsc,
	pcmk_action_t *stonith_op);


	static inline const char *
	pe__rsc_bool_str(const pcmk_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pcmk_resource_t *rsc);
	void group_free(pcmk_resource_t *rsc);
	void clone_free(pcmk_resource_t *rsc);
	void pe__free_bundle(pcmk_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e group_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e clone_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pcmk_resource_t *rsc,
	gboolean current);

	void pe__count_common(pcmk_resource_t *rsc);
	void pe__count_bundle(pcmk_resource_t *rsc);

	void common_free(pcmk_resource_t *rsc);

	pcmk_node_t pe__copy_node(const pcmk_node_t this_node);
	time_t get_effective_time(pcmk_scheduler_t *scheduler);

	/* Failure handling utilities (from failcounts.c) */

	int pe_get_failcount(const pcmk_node_t node, pcmk_resource_t rsc,
	time_t *last_failure, uint32_t flags,
	const xmlNode *xml_op);

	pcmk_action_t pe__clear_failcount(pcmk_resource_t rsc,
	const pcmk_node_t node, const char reason,
	pcmk_scheduler_t *scheduler);

	/* Functions for finding/counting a resource's active nodes */

	bool pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean);

	pcmk_node_t pe__find_active_requires(const pcmk_resource_t rsc,
	unsigned int *count);

	static inline pcmk_node_t *
	pe__current_node(const pcmk_resource_t *rsc)
	{
	return (rsc == NULL)? NULL : rsc->fns->active_node(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	GHashTable pe__node_list2table(const GList list);

	pcmk_action_t get_pseudo_op(const char name, pcmk_scheduler_t *scheduler);
	gboolean order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags);

	void pe__show_node_scores_as(const char file, const char function,
	int line, bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler);

	#define pe__show_node_scores(level, rsc, text, nodes, scheduler) \
	pe__show_node_scores_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (scheduler))

	GHashTable pcmk__unpack_action_meta(pcmk_resource_t rsc,
	const pcmk_node_t *node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config);
	GHashTable pcmk__unpack_action_rsc_params(const xmlNode action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *data_set);
	xmlNode pcmk__find_action_config(const pcmk_resource_t rsc,
	const char *action_name, guint interval_ms,
	bool include_disabled);

	enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc,
	const char *action_name);

	enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc,
	const char *action_name,
	guint interval_ms,
	const char *value);

	enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc,
	const char *action_name,
	enum action_fail_response on_fail,
	GHashTable *meta);

	pcmk_action_t custom_action(pcmk_resource_t rsc, char key, const char task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler);

	# define delete_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), PCMK_ACTION_DELETE, node, \
	optional, rsc->cluster);

	# define stop_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), PCMK_ACTION_STOP, node, \
	optional, rsc->cluster);

	# define reload_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), PCMK_ACTION_START, node, \
	optional, rsc->cluster)

	# define promote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), PCMK_ACTION_PROMOTE, node, \
	optional, rsc->cluster)

	# define demote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), PCMK_ACTION_DEMOTE, node, \
	optional, rsc->cluster)

	extern int pe_get_configured_timeout(pcmk_resource_t rsc, const char action,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t find_first_action(const GList input, const char *uuid,
	const char task, const pcmk_node_t on_node);

	enum action_tasks get_complex_task(const pcmk_resource_t *rsc,
	const char *name);

	GList find_actions(GList input, const char key, const pcmk_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pcmk_node_t *on_node);
	GList pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pcmk_action_t *action);

	void resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler);

	extern int pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default);
	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	gboolean get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role);
	void pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	pcmk_resource_t find_clone_instance(const pcmk_resource_t rsc,
	const char *sub_id);

	extern void destroy_ticket(gpointer data);
	pcmk_ticket_t ticket_new(const char ticket_id, pcmk_scheduler_t *scheduler);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(const pcmk_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return false;
	}

	int pe__target_rc_from_xml(const xmlNode *xml_op);

	gint pe__cmp_node_name(gconstpointer a, gconstpointer b);
	bool is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any);

	pcmk__op_digest_t pe__calculate_digests(pcmk_resource_t rsc, const char *task,
	guint *interval_ms,
	const pcmk_node_t *node,
	const xmlNode *xml_op,
	GHashTable *overrides,
	bool calc_secure,
	pcmk_scheduler_t *scheduler);

	void pe__free_digests(gpointer ptr);

	pcmk__op_digest_t rsc_action_digest_cmp(pcmk_resource_t rsc,
	const xmlNode *xml_op,
	pcmk_node_t *node,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t pe_fence_op(pcmk_node_t node, const char *op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler);
	void trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node,
	const char reason, pcmk_action_t dependency,
	pcmk_scheduler_t *scheduler);

	char pe__action2reason(const pcmk_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite);
	void pe__add_action_expected_result(pcmk_action_t *action, int expected_result);

	void pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler,
	uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	//! \deprecated This function will be removed in a future release
	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay);

	pcmk_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pcmk_scheduler_t scheduler);

	//! \deprecated This function will be removed in a future release
	void common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);
	int pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);

	GList pe__bundle_containers(const pcmk_resource_t bundle);

	int pe__bundle_max(const pcmk_resource_t *rsc);
	bool pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node);
	pcmk_resource_t pe__bundled_resource(const pcmk_resource_t rsc);
	const pcmk_resource_t pe__get_rsc_in_container(const pcmk_resource_t instance);
	pcmk_resource_t pe__first_container(const pcmk_resource_t bundle);
	void pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data);
	void pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data);
	pcmk_resource_t pe__find_bundle_replica(const pcmk_resource_t bundle,
	const pcmk_node_t *node);
	bool pe__bundle_needs_remote_name(pcmk_resource_t *rsc);
	const char pe__add_bundle_remote_name(pcmk_resource_t rsc,
	pcmk_scheduler_t *scheduler,
	xmlNode xml, const char field);

	const char pe__node_attribute_calculated(const pcmk_node_t node,
	const char *name,
	const pcmk_resource_t *rsc,
	enum pcmk__rsc_node node_type,
	bool force_host);
	const char pe_node_attribute_raw(const pcmk_node_t node, const char *name);
	bool pe__is_universal_clone(const pcmk_resource_t *rsc,
	const pcmk_scheduler_t *scheduler);
	void pe__add_param_check(const xmlNode rsc_op, pcmk_resource_t rsc,
	pcmk_node_t *node, enum pcmk__check_parameters,
	pcmk_scheduler_t *scheduler);
	void pe__foreach_param_check(pcmk_scheduler_t *scheduler,
	void (cb)(pcmk_resource_t, pcmk_node_t*,
	const xmlNode*,
	enum pcmk__check_parameters));
	void pe__free_param_checks(pcmk_scheduler_t *scheduler);

	bool pe__shutdown_requested(const pcmk_node_t *node);
	void pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*
	* \param[in,out] out Output object to register messages with
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite,
	pcmk_scheduler_t *scheduler);

	bool pe__resource_is_disabled(const pcmk_resource_t *rsc);
	void pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node);

	GList pe__rscs_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	GList pe__unames_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	bool pe__rsc_has_tag(pcmk_scheduler_t scheduler, const char rsc,
	const char *tag);
	bool pe__uname_has_tag(pcmk_scheduler_t scheduler, const char node,
	const char *tag);

	bool pe__rsc_running_on_only(const pcmk_resource_t *rsc,
	const pcmk_node_t *node);
	bool pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s);
	GList * pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s);

	bool pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__clone_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__group_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char *name);

	const char pe__clone_child_id(const pcmk_resource_t rsc);

	int pe__sum_node_health_scores(const pcmk_node_t *node, int base_health);
	int pe__node_health(pcmk_node_t *node);

	static inline enum pcmk__health_strategy
	pe__health_strategy(pcmk_scheduler_t *scheduler)
	{
	return pcmk__parse_health_strategy(pe_pref(scheduler->config_hash,
	PCMK__OPT_NODE_HEALTH_STRATEGY));
	}

	static inline int
	pe__health_score(const char option, pcmk_scheduler_t scheduler)
	{
	return char2score(pe_pref(scheduler->config_hash, option));
	}

	/*!
	* \internal
	* \brief Return a string suitable for logging as a node name
	*
	* \param[in] node Node to return a node name string for
	*
	* \return Node name if available, otherwise node ID if available,
	* otherwise "unspecified node" if node is NULL or "unidentified node"
	* if node has neither a name nor ID.
	*/
	static inline const char *
	pe__node_name(const pcmk_node_t *node)
	{
	if (node == NULL) {
	return "unspecified node";

	} else if (node->details->uname != NULL) {
	return node->details->uname;

	} else if (node->details->id != NULL) {
	return node->details->id;

	} else {
	return "unidentified node";
	}
	}

	/*!
	* \internal
	* \brief Check whether two node objects refer to the same node
	*
	* \param[in] node1 First node object to compare
	* \param[in] node2 Second node object to compare
	*
	* \return true if \p node1 and \p node2 refer to the same node
	*/
	static inline bool
	pe__same_node(const pcmk_node_t node1, const pcmk_node_t node2)
	{
	return (node1 != NULL) && (node2 != NULL)
	&& (node1->details == node2->details);
	}

	/*!
	* \internal
	* \brief Get the operation key from an action history entry
	*
	* \param[in] xml Action history entry
	*
	* \return Entry's operation key
	*/
	static inline const char *
	pe__xe_history_key(const xmlNode *xml)
	{
	if (xml == NULL) {
	return NULL;
	} else {
	/* @COMPAT Pacemaker <= 1.1.5 did not add the key, and used the ID
	* instead. Checking for that allows us to process old saved CIBs,
	* including some regression tests.
	*/
	const char *key = crm_element_value(xml, XML_LRM_ATTR_TASK_KEY);

	return pcmk__str_empty(key)? ID(xml) : key;
	}
	}

	#endif
	diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
	index 2d8873ab24..006a97b6e8 100644
	--- a/lib/pacemaker/pcmk_graph_producer.c
	+++ b/lib/pacemaker/pcmk_graph_producer.c
	@@ -1,1096 +1,1096 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, XML_ATTR_ID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pcmk_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Count (optionally add to XML) nodes needing maintenance state update
	*
	* \param[in,out] xml Parent XML tag to add to, if any
	* \param[in] scheduler Scheduler data
	*
	* \return Count of nodes added
	* \note Only Pacemaker Remote nodes are considered currently
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pcmk_scheduler_t scheduler)
	{
	xmlNode *maintenance = NULL;
	int count = 0;

	if (xml != NULL) {
	maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
	}
	for (const GList *iter = scheduler->nodes;
	iter != NULL; iter = iter->next) {
	const pcmk_node_t *node = iter->data;

	if (pe__is_guest_or_remote_node(node) &&
	(node->details->maintenance != node->details->remote_maintenance)) {

	if (maintenance != NULL) {
	crm_xml_add(add_node_to_xml_by_id(node->details->id,
	maintenance),
	XML_NODE_IS_MAINTENANCE,
	(node->details->maintenance? "1" : "0"));
	}
	count++;
	}
	}
	crm_trace("%s %d nodes in need of maintenance mode update in state",
	((maintenance == NULL)? "Counted" : "Added"), count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_maintenance_update(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, scheduler) != 0) {
	action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler);
	pe__set_action_flags(action, pcmk_action_always_in_graph);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	*/
	static void
	add_downed_nodes(xmlNode xml, const pcmk_action_t action)
	{
	CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
	return);

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__is_fencing_action(fence)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(action->node->details->data_set,
	action->node, add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pcmk_action_t *input;
	bool migrating = false;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pcmk__related_action_t *) iter->data)->action;
	if ((input->rsc != NULL)
	&& pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
	&& pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	migrating = true;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph operation key for a clone action
	*
	* \param[in] action Clone action
	* \param[in] interval_ms Action interval in milliseconds
	*
	* \return Newly allocated string with transition graph operation key
	*/
	static char *
	clone_op_key(const pcmk_action_t *action, guint interval_ms)
	{
	if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_operation");

	CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
	return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);

	} else if (action->cancel_task != NULL) {
	return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
	interval_ms);
	} else {
	return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
	}
	}

	/*!
	* \internal
	* \brief Add node details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] xml Transition graph action XML for \p action
	*/
	static void
	add_node_details(const pcmk_action_t action, xmlNode xml)
	{
	pcmk_node_t *router_node = pcmk__connection_host_for_action(action);

	crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
	crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
	if (router_node != NULL) {
	crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Add resource details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_resource_details(const pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER,
	XML_ATTR_TYPE
	};

	/* If a resource is locked to a node via shutdown-lock, mark its actions
	* so the controller can preserve the lock when the action completes.
	*/
	if (pcmk__action_locks_rsc_to_node(action)) {
	crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	(const char *) action->rsc->xml->name);
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed)
	&& (action->rsc->clone_name != NULL)) {
	/* Use the numbered instance name here, because if there is more
	* than one instance on a node, we need to make sure the command
	* goes to the right one.
	*
	* This is important even for anonymous clones, because the clone's
	* unique meta-attribute might have just been toggled from on to
	* off.
	*/
	crm_debug("Using orphan clone name %s instead of %s",
	action->rsc->id, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka %s)",
	xml_id, action->rsc->id, action->rsc->clone_name);

	/* ID is what we'd like client to use
	* ID_LONG is what they might know it as instead
	*
	* ID_LONG is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
	if ((action->rsc->clone_name != NULL)
	&& !pcmk__str_eq(xml_id, action->rsc->clone_name,
	pcmk__str_none)) {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
	}

	for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/*!
	* \internal
	* \brief Add action attributes to transition graph action XML
	*
	* \param[in,out] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_action_attributes(pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *args_xml = NULL;

	/* We create free-standing XML to start, so we can sort the attributes
	* before adding it to action_xml, which keeps the scheduler regression
	* test graphs comparable.
	*/
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);

	crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	g_hash_table_foreach(action->extra, hash2field, args_xml);

	if ((action->rsc != NULL) && (action->node != NULL)) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node,
	action->rsc->cluster);

	pcmk__substitute_remote_addr(action->rsc, params);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	} else if ((action->rsc != NULL)
	&& (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL,
	action->rsc->cluster);

	g_hash_table_foreach(params, hash2smartfield, args_xml);
	}

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	pcmk_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->add_graph_meta(parent, args_xml);
	parent = parent->parent;
	}

	pcmk__add_bundle_meta_to_xml(args_xml, action);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& (action->node != NULL)) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield,
	args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);
	}

	/*!
	* \internal
	* \brief Create the transition graph XML for a scheduled action
	*
	* \param[in,out] parent Parent XML element to add action to
	* \param[in,out] action Scheduled action
	* \param[in] skip_details If false, add action details as sub-elements
	* \param[in] scheduler Scheduler data
	*/
	static void
	create_graph_action(xmlNode parent, pcmk_action_t action, bool skip_details,
	const pcmk_scheduler_t *scheduler)
	{
	bool needs_node_info = true;
	bool needs_maintenance_info = false;
	xmlNode *action_xml = NULL;

	if ((action == NULL) \|\| (scheduler == NULL)) {
	return;
	}

	// Create the top-level element based on task

	if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
	/* All fences need node info; guest node fences are pseudo-events */
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	}

	} else if (pcmk__str_any_of(action->task,
	PCMK_ACTION_DO_SHUTDOWN,
	PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
	pcmk__str_none)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
	pcmk__str_none)) {
	needs_maintenance_info = true;
	}
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = false;

	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
	}

	crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);

	if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}
	clone_key = clone_op_key(action, interval_ms);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY,
	action->uuid);
	free(clone_key);
	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if (needs_node_info && (action->node != NULL)) {
	add_node_details(action, action_xml);
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
	strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
	strdup(action->node->details->id));
	}

	if (skip_details) {
	return;
	}

	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_pseudo)) {

	// This is a real resource action, so add resource details
	add_resource_details(action, action_xml);
	}

	/* List any attributes in effect */
	add_action_attributes(action, action_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Check whether an action should be added to the transition graph
	*
	* \param[in] action Action to check
	*
	* \return true if action should be added to graph, otherwise false
	*/
	static bool
	should_add_action_to_graph(const pcmk_action_t *action)
	{
	if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_optional)
	&& !pcmk_is_set(action->flags, pcmk_action_always_in_graph)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;
	}

	/* Actions for unmanaged resources should be excluded from the graph,
	* with the exception of monitors and cancellation of recurring monitors.
	*/
	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {

	const char *interval_ms_s;

	/* A cancellation of a recurring monitor will get here because the task
	* is cancel rather than monitor, but the interval can still be used to
	* recognize it. The interval has been normalized to milliseconds by
	* this point, so a string comparison is sufficient.
	*/
	interval_ms_s = g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	/* Always add pseudo-actions, fence actions, and shutdown actions (already
	* determined to be required and runnable by this point)
	*/
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)
	\|\| pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
	PCMK_ACTION_DO_SHUTDOWN, NULL)) {
	return true;
	}

	if (action->node == NULL) {
	- pe_err("Skipping action %s (%d) "
	- "because it was not assigned to a node (bug?)",
	- action->uuid, action->id);
	+ pcmk__sched_err("Skipping action %s (%d) "
	+ "because it was not assigned to a node (bug?)",
	+ action->uuid, action->id);
	pcmk__log_action("Unassigned", action, false);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_on_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (!action->node->details->online) {
	- pe_err("Skipping action %s (%d) "
	- "because it was scheduled for offline node (bug?)",
	- action->uuid, action->id);
	+ pcmk__sched_err("Skipping action %s (%d) "
	+ "because it was scheduled for offline node (bug?)",
	+ action->uuid, action->id);
	pcmk__log_action("Offline node", action, false);
	return false;

	} else if (action->node->details->unclean) {
	- pe_err("Skipping action %s (%d) "
	- "because it was scheduled for unclean node (bug?)",
	- action->uuid, action->id);
	+ pcmk__sched_err("Skipping action %s (%d) "
	+ "because it was scheduled for unclean node (bug?)",
	+ action->uuid, action->id);
	pcmk__log_action("Unclean node", action, false);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering's flags can change an action
	*
	* \param[in] ordering Ordering to check
	*
	* \return true if ordering has flags that can change an action, false otherwise
	*/
	static bool
	ordering_can_change_actions(const pcmk__related_action_t *ordering)
	{
	return pcmk_any_flags_set(ordering->type,
	~(pcmk__ar_then_implies_first_graphed
	\|pcmk__ar_first_implies_then_graphed
	\|pcmk__ar_ordered));
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	should_add_input_to_graph(const pcmk_action_t *action,
	pcmk__related_action_t *input)
	{
	if (input->state == pe_link_dumped) {
	return true;
	}

	if ((uint32_t) input->type == pcmk__ar_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& !ordering_can_change_actions(input)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& pcmk_is_set(input->type, pcmk__ar_min_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"minimum number of instances required but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_unmigratable_then_blocks)
	&& !pcmk_is_set(input->action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input blocked if 'then' unmigratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_if_first_unmigratable)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): ordering applies "
	"only if input is unmigratable, but it is migratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (((uint32_t) input->type == pcmk__ar_ordered)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) {
	pcmk_node_t *input_node = input->action->node;

	if ((action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {

	pcmk_node_t *assigned = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where
	* the resource has been assigned, not where migrate_to will be
	* executed.
	*/
	if (!pe__same_node(input_node, assigned)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"migration target %s is not same as input node %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(assigned? assigned->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if (!pe__same_node(input_node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if ((uint32_t) input->type == pcmk__ar_if_required_on_same_node) {
	if (input->action->node && action->node
	&& !pe__same_node(input->action->node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	pe__node_name(action->node),
	pe__node_name(input->action->node));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pcmk_action_always_in_graph
	\|pcmk_action_added_to_graph)
	&& !should_add_action_to_graph(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
	action->uuid, action->id, action_type_str(input->action->flags),
	input->action->uuid, input->action->id,
	action_node_str(input->action),
	action_runnable_str(input->action->flags),
	action_optional_str(input->action->flags), input->type);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering creates an ordering loop
	*
	* \param[in] init_action "First" action in ordering
	* \param[in] action Callers should always set this the same as
	* \p init_action (this function may use a different
	* value for recursive calls)
	* \param[in,out] input Action wrapper for "then" action in ordering
	*
	* \return true if the ordering creates a loop, otherwise false
	*/
	bool
	pcmk__graph_has_loop(const pcmk_action_t *init_action,
	const pcmk_action_t action, pcmk__related_action_t input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pcmk_action_detect_loop)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!should_add_input_to_graph(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pe__set_action_flags(input->action, pcmk_action_detect_loop);

	crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (pcmk__graph_has_loop(init_action, input->action,
	(pcmk__related_action_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	break;
	}
	}

	pe__clear_action_flags(input->action, pcmk_action_detect_loop);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	/*!
	* \internal
	* \brief Create a synapse XML element for a transition graph
	*
	* \param[in] action Action that synapse is for
	* \param[in,out] scheduler Scheduler data containing graph
	*
	* \return Newly added XML element for new graph synapse
	*/
	static xmlNode *
	create_graph_synapse(const pcmk_action_t action, pcmk_scheduler_t scheduler)
	{
	int synapse_priority = 0;
	xmlNode *syn = create_xml_node(scheduler->graph, "synapse");

	crm_xml_add_int(syn, XML_ATTR_ID, scheduler->num_synapse);
	scheduler->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}
	return syn;
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in,out] data Action to possibly add
	* \param[in,out] user_data Scheduler data
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pcmk__related_action_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after
	* pcmk__apply_orderings() is complete, and nothing after this should rely
	* on those type flags. (For example, some code looks for type equal to
	* some flag rather than whether the flag is set, and some code looks for
	* particular combinations of flags -- such code must be done before
	* pcmk__create_graph().)
	*/
	static void
	add_action_to_graph(gpointer data, gpointer user_data)
	{
	pcmk_action_t action = (pcmk_action_t ) data;
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) user_data;

	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;

	/* If we haven't already, de-duplicate inputs (even if we won't be adding
	* the action to the graph, so that crm_simulate's dot graphs don't have
	* duplicates).
	*/
	if (!pcmk_is_set(action->flags, pcmk_action_inputs_deduplicated)) {
	pcmk__deduplicate_action_inputs(action);
	pe__set_action_flags(action, pcmk_action_inputs_deduplicated);
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	\|\| !should_add_action_to_graph(action)) {
	return; // Already added, or shouldn't be
	}
	pe__set_action_flags(action, pcmk_action_added_to_graph);

	crm_trace("Adding action %d (%s%s%s) to graph",
	action->id, action->uuid,
	((action->node == NULL)? "" : " on "),
	((action->node == NULL)? "" : action->node->details->uname));

	syn = create_graph_synapse(action, scheduler);
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	create_graph_action(set, action, false, scheduler);

	for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *input = lpc->data;

	if (should_add_input_to_graph(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	create_graph_action(input_xml, input->action, true, scheduler);
	}
	}
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error \|\| crm_config_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning \|\| crm_config_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc)
	{
	GList *iter = NULL;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);

	// First add the resource's own actions
	g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);

	// Then recursively add its children's actions (appropriate to variant)
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	child_rsc->cmds->add_actions_to_graph(child_rsc);
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph with all cluster actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_graph(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d", transition_id);

	scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(scheduler->config_hash, "cluster-delay");
	crm_xml_add(scheduler->graph, "cluster-delay", value);

	value = pe_pref(scheduler->config_hash, "stonith-timeout");
	crm_xml_add(scheduler->graph, "stonith-timeout", value);

	crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_xml_add(scheduler->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(scheduler->graph, "failed-start-offset", "1");
	}

	value = pe_pref(scheduler->config_hash, "batch-limit");
	crm_xml_add(scheduler->graph, "batch-limit", value);

	crm_xml_add_int(scheduler->graph, "transition_id", transition_id);

	value = pe_pref(scheduler->config_hash, "migration-limit");
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(scheduler->graph, "migration-limit", value);
	}

	if (scheduler->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) scheduler->recheck_by);
	crm_xml_add(scheduler->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	// Add resource actions to graph
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id);
	rsc->cmds->add_actions_to_graph(rsc);
	}

	// Add pseudo-action for list of nodes with maintenance state update
	add_maintenance_update(scheduler);

	// Add non-resource (node) actions
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	if ((action->rsc != NULL)
	&& (action->node != NULL)
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_runnable)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* Eventually we should just ignore the 'fence' case, but for now
	* it's the best way to detect (in CTS) when CIB resource updates
	* are being lost.
	*/
	if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	\|\| (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) {
	const bool managed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_managed);
	const bool failed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_failed);

	crm_crit("Cannot %s %s because of %s:%s%s (%s)",
	action->node->details->unclean? "fence" : "shut down",
	pe__node_name(action->node), action->rsc->id,
	(managed? " blocked" : " unmanaged"),
	(failed? " failed" : ""), action->uuid);
	}
	}

	add_action_to_graph((gpointer) action, (gpointer) scheduler);
	}

	crm_log_xml_trace(scheduler->graph, "graph");
	}
	diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
	index 0d1beb9771..1177f9f3d2 100644
	--- a/lib/pacemaker/pcmk_sched_constraints.c
	+++ b/lib/pacemaker/pcmk_sched_constraints.c
	@@ -1,424 +1,424 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdbool.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	static bool
	evaluate_lifetime(xmlNode lifetime, pcmk_scheduler_t scheduler)
	{
	bool result = FALSE;
	crm_time_t *next_change = crm_time_new_undefined();

	result = pe_evaluate_rules(lifetime, NULL, scheduler->now, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "constraint lifetime");
	}
	crm_time_free(next_change);
	return result;
	}

	/*!
	* \internal
	* \brief Unpack constraints from XML
	*
	* Given scheduler data, unpack all constraints from its input XML into
	* data structures.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_constraints(pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_constraints = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS);

	for (xmlNode *xml_obj = pcmk__xe_first_child(xml_constraints);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char tag = (const char ) xml_obj->name;

	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without "
	XML_ATTR_ID, tag);
	continue;
	}

	crm_trace("Unpacking %s constraint '%s'", tag, id);

	lifetime = first_named_child(xml_obj, "lifetime");
	if (lifetime != NULL) {
	pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
	"deprecated (the rules it contains should "
	"instead be direct descendants of the "
	"constraint object)", id);
	}

	if ((lifetime != NULL) && !evaluate_lifetime(lifetime, scheduler)) {
	crm_info("Constraint %s %s is not active", tag, id);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_ORDER, tag, pcmk__str_none)) {
	pcmk__unpack_ordering(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, tag, pcmk__str_none)) {
	pcmk__unpack_colocation(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, tag,
	pcmk__str_none)) {
	pcmk__unpack_location(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_TICKET, tag, pcmk__str_none)) {
	pcmk__unpack_rsc_ticket(xml_obj, scheduler);

	} else {
	- pe_err("Unsupported constraint type: %s", tag);
	+ pcmk__config_err("Unsupported constraint type: %s", tag);
	}
	}
	}

	pcmk_resource_t *
	pcmk__find_constraint_resource(GList rsc_list, const char id)
	{
	if (id == NULL) {
	return NULL;
	}
	for (GList *iter = rsc_list; iter != NULL; iter = iter->next) {
	pcmk_resource_t *parent = iter->data;
	pcmk_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
	pcmk_rsc_match_history);

	if (match != NULL) {
	if (!pcmk__str_eq(match->id, id, pcmk__str_none)) {
	/* We found an instance of a clone instead */
	match = uber_parent(match);
	crm_debug("Found %s for %s", match->id, id);
	}
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether an ID references a resource tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id Tag ID to search for
	* \param[out] tag Where to store tag, if found
	*
	* \return true if ID refers to a tagged resource or resource set template,
	* otherwise false
	*/
	static bool
	find_constraint_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_tag_t **tag)
	{
	*tag = NULL;

	// Check whether id refers to a resource set template
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	crm_warn("No resource is derived from template '%s'", id);
	return false;
	}
	return true;
	}

	// If not, check whether id refers to a tag
	if (g_hash_table_lookup_extended(scheduler->tags, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	crm_warn("No resource is tagged with '%s'", id);
	return false;
	}
	return true;
	}

	crm_warn("No template or tag named '%s'", id);
	return false;
	}

	/*!
	* \brief
	* \internal Check whether an ID refers to a valid resource or tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id ID to search for
	* \param[out] rsc Where to store resource, if found
	* (or NULL to skip searching resources)
	* \param[out] tag Where to store tag, if found
	* (or NULL to skip searching tags)
	*
	* \return true if id refers to a resource (possibly indirectly via a tag)
	*/
	bool
	pcmk__valid_resource_or_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_resource_t rsc, pcmk_tag_t tag)
	{
	if (rsc != NULL) {
	*rsc = pcmk__find_constraint_resource(scheduler->resources, id);
	if (*rsc != NULL) {
	return true;
	}
	}

	if ((tag != NULL) && find_constraint_tag(scheduler, id, tag)) {
	return true;
	}

	return false;
	}

	/*!
	* \internal
	* \brief Replace any resource tags with equivalent resource_ref entries
	*
	* If a given constraint has resource sets, check each set for resource_ref
	* entries that list tags rather than resource IDs, and replace any found with
	* resource_ref entries for the corresponding resource IDs.
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[in] scheduler Scheduler data
	*
	* \return Equivalent XML with resource tags replaced (or NULL if none)
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	pcmk__expand_tags_in_sets(xmlNode xml_obj, const pcmk_scheduler_t scheduler)
	{
	xmlNode *new_xml = NULL;
	bool any_refs = false;

	// Short-circuit if there are no sets
	if (first_named_child(xml_obj, XML_CONS_TAG_RSC_SET) == NULL) {
	return NULL;
	}

	new_xml = copy_xml(xml_obj);

	for (xmlNode *set = first_named_child(new_xml, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	GList *tag_refs = NULL;
	GList *iter = NULL;

	for (xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	if (!pcmk__valid_resource_or_tag(scheduler, ID(xml_rsc), &rsc,
	&tag)) {
	pcmk__config_err("Ignoring resource sets for constraint '%s' "
	"because '%s' is not a valid resource or tag",
	ID(xml_obj), ID(xml_rsc));
	free_xml(new_xml);
	return NULL;

	} else if (rsc) {
	continue;

	} else if (tag) {
	// resource_ref under resource_set references template or tag
	xmlNode *last_ref = xml_rsc;

	/* For example, given the original XML:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*
	* If rsc2 and rsc3 are tagged with tag1, we add them after it:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc2"/>
	* <resource_ref id="rsc3"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*/

	for (iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *new_rsc_ref = NULL;

	new_rsc_ref = xmlNewDocRawNode(set->doc, NULL,
	(pcmkXmlStr)
	XML_TAG_RESOURCE_REF,
	NULL);
	crm_xml_add(new_rsc_ref, XML_ATTR_ID, obj_ref);
	xmlAddNextSibling(last_ref, new_rsc_ref);

	last_ref = new_rsc_ref;
	}

	any_refs = true;

	/* Freeing the resource_ref now would break the XML child
	* iteration, so just remember it for freeing later.
	*/
	tag_refs = g_list_append(tag_refs, xml_rsc);
	}
	}

	/* Now free '<resource_ref id="tag1"/>', and finally get:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="rsc2"/>
	<resource_ref id="rsc3"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	*/
	for (iter = tag_refs; iter != NULL; iter = iter->next) {
	xmlNode *tag_ref = iter->data;

	free_xml(tag_ref);
	}
	g_list_free(tag_refs);
	}

	if (!any_refs) {
	free_xml(new_xml);
	new_xml = NULL;
	}
	return new_xml;
	}

	/*!
	* \internal
	* \brief Convert a tag into a resource set of tagged resources
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[out] rsc_set Where to store resource set XML
	* \param[in] attr Name of XML attribute with resource or tag ID
	* \param[in] convert_rsc If true, convert to set even if \p attr
	* references a resource
	* \param[in] scheduler Scheduler data
	*/
	bool
	pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, const pcmk_scheduler_t *scheduler)
	{
	const char *cons_id = NULL;
	const char *id = NULL;

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	*rsc_set = NULL;

	CRM_CHECK((xml_obj != NULL) && (attr != NULL), return false);

	cons_id = ID(xml_obj);
	if (cons_id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return false;
	}

	id = crm_element_value(xml_obj, attr);
	if (id == NULL) {
	return true;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", cons_id, id);
	return false;

	} else if (tag) {
	/* The "attr" attribute (for a resource in a constraint) specifies a
	* template or tag. Add the corresponding resource_set containing the
	* resources derived from or tagged with it.
	*/
	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	for (GList *iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *rsc_ref = NULL;

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, obj_ref);
	}

	/* Set sequential="false" for the resource_set */
	pcmk__xe_set_bool_attr(*rsc_set, "sequential", false);

	} else if ((rsc != NULL) && convert_rsc) {
	/* Even if a regular resource is referenced by "attr", convert it into a
	* resource_set, because the other resource reference in the constraint
	* could be a template or tag.
	*/
	xmlNode *rsc_ref = NULL;

	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, XML_ATTR_ID, id);

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, XML_ATTR_ID, id);

	} else {
	return true;
	}

	/* Remove the "attr" attribute referencing the template/tag */
	if (*rsc_set != NULL) {
	xml_remove_prop(xml_obj, attr);
	}

	return true;
	}

	/*!
	* \internal
	* \brief Create constraints inherent to resource types
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_internal_constraints(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Create internal constraints");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->internal_constraints(rsc);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c
	index 8d80fa0ff4..c1e5945979 100644
	--- a/lib/pacemaker/pcmk_sched_location.c
	+++ b/lib/pacemaker/pcmk_sched_location.c
	@@ -1,675 +1,675 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static int
	get_node_score(const char rule, const char score, bool raw,
	pcmk_node_t node, pcmk_resource_t rsc)
	{
	int score_f = 0;

	if (score == NULL) {
	- pe_err("Rule %s: no score specified. Assuming 0.", rule);
	+ pcmk__config_err("Rule %s: no score specified (assuming 0)", rule);

	} else if (raw) {
	score_f = char2score(score);

	} else {
	const char *attr_score = NULL;

	attr_score = pe__node_attribute_calculated(node, score, rsc,
	pcmk__rsc_node_current,
	false);

	if (attr_score == NULL) {
	crm_debug("Rule %s: %s did not have a value for %s",
	rule, pe__node_name(node), score);
	score_f = -INFINITY;

	} else {
	crm_debug("Rule %s: %s had value %s for %s",
	rule, pe__node_name(node), attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}

	static pcmk__location_t *
	generate_location_rule(pcmk_resource_t rsc, xmlNode rule_xml,
	const char discovery, crm_time_t next_change,
	pe_re_match_data_t *re_match_data)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role = NULL;

	GList *iter = NULL;
	GList *nodes = NULL;

	bool do_and = true;
	bool accept = true;
	bool raw_score = true;
	bool score_allocated = false;

	pcmk__location_t *location_rule = NULL;

	rule_xml = expand_idref(rule_xml, rsc->cluster->input);
	if (rule_xml == NULL) {
	return NULL;
	}

	rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
	boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
	role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);

	crm_trace("Processing rule: %s", rule_id);

	if ((role != NULL) && (text2role(role) == pcmk_role_unknown)) {
	- pe_err("Bad role specified for %s: %s", rule_id, role);
	+ pcmk__config_err("Bad role specified for %s: %s", rule_id, role);
	return NULL;
	}

	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
	if (score == NULL) {
	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
	if (score != NULL) {
	raw_score = false;
	}
	}
	if (pcmk__str_eq(boolean, "or", pcmk__str_casei)) {
	do_and = false;
	}

	location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL);

	if (location_rule == NULL) {
	return NULL;
	}

	if ((re_match_data != NULL) && (re_match_data->nregs > 0)
	&& (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {

	char *result = pe_expand_re_matches(score, re_match_data);

	if (result != NULL) {
	score = result;
	score_allocated = true;
	}
	}

	if (role != NULL) {
	crm_trace("Setting role filter: %s", role);
	location_rule->role_filter = text2role(role);
	if (location_rule->role_filter == pcmk_role_unpromoted) {
	/* Any promotable clone cannot be promoted without being in the
	* unpromoted role first. Ergo, any constraint for the unpromoted
	* role applies to every role.
	*/
	location_rule->role_filter = pcmk_role_unknown;
	}
	}
	if (do_and) {
	nodes = pcmk__copy_node_list(rsc->cluster->nodes, true);
	for (iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;

	node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
	}
	}

	for (iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	int score_f = 0;
	pcmk_node_t *node = iter->data;
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = pe_rsc_params(rsc, node, rsc->cluster),
	.meta = rsc->meta,
	};

	accept = pe_test_rule(rule_xml, node->details->attrs, pcmk_role_unknown,
	rsc->cluster->now, next_change, &match_data);

	crm_trace("Rule %s %s on %s", ID(rule_xml), accept? "passed" : "failed",
	pe__node_name(node));

	score_f = get_node_score(rule_id, score, raw_score, node, rsc);

	if (accept) {
	pcmk_node_t *local = pe_find_node_id(nodes, node->details->id);

	if ((local == NULL) && do_and) {
	continue;

	} else if (local == NULL) {
	local = pe__copy_node(node);
	nodes = g_list_append(nodes, local);
	}

	if (!do_and) {
	local->weight = pcmk__add_scores(local->weight, score_f);
	}
	crm_trace("%s has score %s after %s", pe__node_name(node),
	pcmk_readable_score(local->weight), rule_id);

	} else if (do_and && !accept) {
	// Remove it
	pcmk_node_t *delete = pe_find_node_id(nodes, node->details->id);

	if (delete != NULL) {
	nodes = g_list_remove(nodes, delete);
	crm_trace("%s did not match", pe__node_name(node));
	}
	free(delete);
	}
	}

	if (score_allocated) {
	free((char *)score);
	}

	location_rule->nodes = nodes;
	if (location_rule->nodes == NULL) {
	crm_trace("No matching nodes for rule %s", rule_id);
	return NULL;
	}

	crm_trace("%s: %d nodes matched",
	rule_id, g_list_length(location_rule->nodes));
	return location_rule;
	}

	static void
	unpack_rsc_location(xmlNode xml_obj, pcmk_resource_t rsc, const char *role,
	const char score, pe_re_match_data_t re_match_data)
	{
	pcmk__location_t *location = NULL;
	const char *rsc_id = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
	const char *discovery = crm_element_value(xml_obj,
	XML_LOCATION_ATTR_DISCOVERY);

	if (rsc == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, rsc_id);
	return;
	}

	if (score == NULL) {
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	}

	if ((node != NULL) && (score != NULL)) {
	int score_i = char2score(score);
	pcmk_node_t *match = pe_find_node(rsc->cluster->nodes, node);

	if (!match) {
	return;
	}
	location = pcmk__new_location(id, rsc, score_i, discovery, match);

	} else {
	bool empty = true;
	crm_time_t *next_change = crm_time_new_undefined();

	/* This loop is logically parallel to pe_evaluate_rules(), except
	* instead of checking whether any rule is active, we set up location
	* constraints for each active rule.
	*/
	for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
	rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
	empty = false;
	crm_trace("Unpacking %s/%s", id, ID(rule_xml));
	generate_location_rule(rsc, rule_xml, discovery, next_change,
	re_match_data);
	}

	if (empty) {
	pcmk__config_err("Ignoring constraint '%s' because it contains "
	"no rules", id);
	}

	/* If there is a point in the future when the evaluation of a rule will
	* change, make sure the scheduler is re-run by that time.
	*/
	if (crm_time_is_defined(next_change)) {
	time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(t, rsc->cluster,
	"location rule evaluation");
	}
	crm_time_free(next_change);
	return;
	}

	if (role == NULL) {
	role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
	}

	if ((location != NULL) && (role != NULL)) {
	if (text2role(role) == pcmk_role_unknown) {
	- pe_err("Invalid constraint %s: Bad role %s", id, role);
	+ pcmk__config_err("Invalid constraint %s: Bad role %s", id, role);
	return;

	} else {
	enum rsc_role_e r = text2role(role);
	switch (r) {
	case pcmk_role_unknown:
	case pcmk_role_started:
	case pcmk_role_unpromoted:
	/* Applies to all */
	location->role_filter = pcmk_role_unknown;
	break;
	default:
	location->role_filter = r;
	break;
	}
	}
	}
	}

	static void
	unpack_simple_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);

	if (value) {
	pcmk_resource_t *rsc;

	rsc = pcmk__find_constraint_resource(scheduler->resources, value);
	unpack_rsc_location(xml_obj, rsc, NULL, NULL, NULL);
	}

	value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
	if (value) {
	regex_t *r_patt = calloc(1, sizeof(regex_t));
	bool invert = false;

	if (value[0] == '!') {
	value++;
	invert = true;
	}

	if (regcomp(r_patt, value, REG_EXTENDED) != 0) {
	pcmk__config_err("Ignoring constraint '%s' because "
	XML_LOC_ATTR_SOURCE_PATTERN
	" has invalid value '%s'", id, value);
	free(r_patt);
	return;
	}

	for (GList *iter = scheduler->resources; iter != NULL;
	iter = iter->next) {

	pcmk_resource_t *r = iter->data;
	int nregs = 0;
	regmatch_t *pmatch = NULL;
	int status;

	if (r_patt->re_nsub > 0) {
	nregs = r_patt->re_nsub + 1;
	} else {
	nregs = 1;
	}
	pmatch = calloc(nregs, sizeof(regmatch_t));

	status = regexec(r_patt, r->id, nregs, pmatch, 0);

	if (!invert && (status == 0)) {
	pe_re_match_data_t re_match_data = {
	.string = r->id,
	.nregs = nregs,
	.pmatch = pmatch
	};

	crm_debug("'%s' matched '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, &re_match_data);

	} else if (invert && (status != 0)) {
	crm_debug("'%s' is an inverted match of '%s' for %s",
	r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, NULL);

	} else {
	crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
	}

	free(pmatch);
	}

	regfree(r_patt);
	free(r_patt);
	}
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *rsc_id = NULL;
	const char *state = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;
	xmlNode *rsc_set = NULL;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");
	return pcmk_rc_ok;
	}

	rsc_id = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	if (rsc_id == NULL) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, rsc_id);
	return pcmk_rc_unpack_error;

	} else if (rsc != NULL) {
	// No template is referenced
	return pcmk_rc_ok;
	}

	state = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert any template or tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, XML_LOC_ATTR_SOURCE,
	false, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set != NULL) {
	if (state != NULL) {
	// Move "rsc-role" into converted resource_set as "role" attribute
	crm_xml_add(rsc_set, "role", state);
	xml_remove_prop(*expanded_xml, XML_RULE_ATTR_ROLE);
	}
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");

	} else {
	// No sets
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_set(xmlNode location, xmlNode set,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id;
	const char *role;
	const char *local_score;

	CRM_CHECK(set != NULL, return EINVAL);

	set_id = ID(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
	XML_ATTR_ID " in constraint '%s'",
	pcmk__s(ID(location), "(missing ID)"));
	return pcmk_rc_unpack_error;
	}

	role = crm_element_value(set, "role");
	local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	resource = pcmk__find_constraint_resource(scheduler->resources,
	ID(xml_rsc));
	if (resource == NULL) {
	pcmk__config_err("%s: No resource found for %s",
	set_id, ID(xml_rsc));
	return pcmk_rc_unpack_error;
	}

	unpack_rsc_location(location, resource, role, local_score, NULL);
	}

	return pcmk_rc_ok;
	}

	void
	pcmk__unpack_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	bool any_sets = false;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	if (unpack_location_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}

	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	any_sets = true;
	set = expand_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_location_set(xml_obj, set, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml) {
	free_xml(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (!any_sets) {
	unpack_simple_location(xml_obj, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Add a new location constraint to scheduler data
	*
	* \param[in] id XML ID of location constraint
	* \param[in,out] rsc Resource in location constraint
	* \param[in] node_score Constraint score
	* \param[in] discover_mode Resource discovery option for constraint
	* \param[in] node Node in constraint (or NULL if rule-based)
	*
	* \return Newly allocated location constraint
	* \note The result will be added to the cluster (via \p rsc) and should not be
	* freed separately.
	*/
	pcmk__location_t *
	pcmk__new_location(const char id, pcmk_resource_t rsc,
	int node_score, const char discover_mode, pcmk_node_t node)
	{
	pcmk__location_t *new_con = NULL;

	if (id == NULL) {
	- pe_err("Invalid constraint: no ID specified");
	+ pcmk__config_err("Invalid constraint: no ID specified");
	return NULL;

	} else if (rsc == NULL) {
	- pe_err("Invalid constraint %s: no resource specified", id);
	+ pcmk__config_err("Invalid constraint %s: no resource specified", id);
	return NULL;

	} else if (node == NULL) {
	CRM_CHECK(node_score == 0, return NULL);
	}

	new_con = calloc(1, sizeof(pcmk__location_t));
	if (new_con != NULL) {
	new_con->id = strdup(id);
	new_con->rsc = rsc;
	new_con->nodes = NULL;
	new_con->role_filter = pcmk_role_unknown;

	if (pcmk__str_eq(discover_mode, "always",
	pcmk__str_null_matches\|pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_always;

	} else if (pcmk__str_eq(discover_mode, "never", pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_never;

	} else if (pcmk__str_eq(discover_mode, "exclusive", pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_exclusive;
	rsc->exclusive_discover = TRUE;

	} else {
	- pe_err("Invalid " XML_LOCATION_ATTR_DISCOVERY " value %s "
	- "in location constraint", discover_mode);
	+ pcmk__config_err("Invalid " XML_LOCATION_ATTR_DISCOVERY " value %s "
	+ "in location constraint", discover_mode);
	}

	if (node != NULL) {
	pcmk_node_t *copy = pe__copy_node(node);

	copy->weight = node_score;
	new_con->nodes = g_list_prepend(NULL, copy);
	}

	rsc->cluster->placement_constraints = g_list_prepend(
	rsc->cluster->placement_constraints, new_con);
	rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
	}

	return new_con;
	}

	/*!
	* \internal
	* \brief Apply all location constraints
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__apply_locations(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->placement_constraints;
	iter != NULL; iter = iter->next) {
	pcmk__location_t *location = iter->data;

	location->rsc->cmds->apply_location(location->rsc, location);
	}
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node scores
	*
	* \param[in,out] rsc Resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*
	* \note This does not consider the resource's children, so the resource's
	* apply_location() method should be used instead in most cases.
	*/
	void
	pcmk__apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	bool need_role = false;

	CRM_ASSERT((rsc != NULL) && (location != NULL));

	// If a role was specified, ensure constraint is applicable
	need_role = (location->role_filter > pcmk_role_unknown);
	if (need_role && (location->role_filter != rsc->next_role)) {
	pcmk__rsc_trace(rsc,
	"Not applying %s to %s because role will be %s not %s",
	location->id, rsc->id, role2text(rsc->next_role),
	role2text(location->role_filter));
	return;
	}

	if (location->nodes == NULL) {
	pcmk__rsc_trace(rsc, "Not applying %s to %s because no nodes match",
	location->id, rsc->id);
	return;
	}

	pcmk__rsc_trace(rsc, "Applying %s%s%s to %s", location->id,
	(need_role? " for role " : ""),
	(need_role? role2text(location->role_filter) : ""),
	rsc->id);

	for (GList *iter = location->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;
	pcmk_node_t *allowed_node = g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	if (allowed_node == NULL) {
	pcmk__rsc_trace(rsc, "* = %d on %s",
	node->weight, pe__node_name(node));
	allowed_node = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes,
	(gpointer) allowed_node->details->id,
	allowed_node);
	} else {
	pcmk__rsc_trace(rsc, "* + %d on %s",
	node->weight, pe__node_name(node));
	allowed_node->weight = pcmk__add_scores(allowed_node->weight,
	node->weight);
	}

	if (allowed_node->rsc_discover_mode < location->discover_mode) {
	if (location->discover_mode == pcmk_probe_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	allowed_node->rsc_discover_mode = location->discover_mode;
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_utilization.c b/lib/pacemaker/pcmk_sched_utilization.c
	index 8a0d7a96e6..3aa5fa8b43 100644
	--- a/lib/pacemaker/pcmk_sched_utilization.c
	+++ b/lib/pacemaker/pcmk_sched_utilization.c
	@@ -1,468 +1,469 @@
	/*
	* Copyright 2014-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get integer utilization from a string
	*
	* \param[in] s String representation of a node utilization value
	*
	* \return Integer equivalent of \p s
	* \todo It would make sense to restrict utilization values to nonnegative
	* integers, but the documentation just says "integers" and we didn't
	* restrict them initially, so for backward compatibility, allow any
	* integer.
	*/
	static int
	utilization_value(const char *s)
	{
	int value = 0;

	if ((s != NULL) && (pcmk__scan_min_int(s, &value, INT_MIN) == EINVAL)) {
	- pe_warn("Using 0 for utilization instead of invalid value '%s'", value);
	+ pcmk__config_warn("Using 0 for utilization instead of "
	+ "invalid value '%s'", value);
	value = 0;
	}
	return value;
	}


	/*
	* Functions for comparing node capacities
	*/

	struct compare_data {
	const pcmk_node_t *node1;
	const pcmk_node_t *node2;
	bool node2_only;
	int result;
	};

	/*!
	* \internal
	* \brief Compare a single utilization attribute for two nodes
	*
	* Compare one utilization attribute for two nodes, decrementing the result if
	* the first node has greater capacity, and incrementing it if the second node
	* has greater capacity.
	*
	* \param[in] key Utilization attribute name to compare
	* \param[in] value Utilization attribute value to compare
	* \param[in,out] user_data Comparison data (as struct compare_data*)
	*/
	static void
	compare_utilization_value(gpointer key, gpointer value, gpointer user_data)
	{
	int node1_capacity = 0;
	int node2_capacity = 0;
	struct compare_data *data = user_data;
	const char *node2_value = NULL;

	if (data->node2_only) {
	if (g_hash_table_lookup(data->node1->details->utilization, key)) {
	return; // We've already compared this attribute
	}
	} else {
	node1_capacity = utilization_value((const char *) value);
	}

	node2_value = g_hash_table_lookup(data->node2->details->utilization, key);
	node2_capacity = utilization_value(node2_value);

	if (node1_capacity > node2_capacity) {
	data->result--;
	} else if (node1_capacity < node2_capacity) {
	data->result++;
	}
	}

	/*!
	* \internal
	* \brief Compare utilization capacities of two nodes
	*
	* \param[in] node1 First node to compare
	* \param[in] node2 Second node to compare
	*
	* \return Negative integer if node1 has more free capacity,
	* 0 if the capacities are equal, or a positive integer
	* if node2 has more free capacity
	*/
	int
	pcmk__compare_node_capacities(const pcmk_node_t *node1,
	const pcmk_node_t *node2)
	{
	struct compare_data data = {
	.node1 = node1,
	.node2 = node2,
	.node2_only = false,
	.result = 0,
	};

	// Compare utilization values that node1 and maybe node2 have
	g_hash_table_foreach(node1->details->utilization, compare_utilization_value,
	&data);

	// Compare utilization values that only node2 has
	data.node2_only = true;
	g_hash_table_foreach(node2->details->utilization, compare_utilization_value,
	&data);

	return data.result;
	}


	/*
	* Functions for updating node capacities
	*/

	struct calculate_data {
	GHashTable *current_utilization;
	bool plus;
	};

	/*!
	* \internal
	* \brief Update a single utilization attribute with a new value
	*
	* \param[in] key Name of utilization attribute to update
	* \param[in] value Value to add or substract
	* \param[in,out] user_data Calculation data (as struct calculate_data *)
	*/
	static void
	update_utilization_value(gpointer key, gpointer value, gpointer user_data)
	{
	int result = 0;
	const char *current = NULL;
	struct calculate_data *data = user_data;

	current = g_hash_table_lookup(data->current_utilization, key);
	if (data->plus) {
	result = utilization_value(current) + utilization_value(value);
	} else if (current) {
	result = utilization_value(current) - utilization_value(value);
	}
	g_hash_table_replace(data->current_utilization,
	strdup(key), pcmk__itoa(result));
	}

	/*!
	* \internal
	* \brief Subtract a resource's utilization from node capacity
	*
	* \param[in,out] current_utilization Current node utilization attributes
	* \param[in] rsc Resource with utilization to subtract
	*/
	void
	pcmk__consume_node_capacity(GHashTable *current_utilization,
	const pcmk_resource_t *rsc)
	{
	struct calculate_data data = {
	.current_utilization = current_utilization,
	.plus = false,
	};

	g_hash_table_foreach(rsc->utilization, update_utilization_value, &data);
	}

	/*!
	* \internal
	* \brief Add a resource's utilization to node capacity
	*
	* \param[in,out] current_utilization Current node utilization attributes
	* \param[in] rsc Resource with utilization to add
	*/
	void
	pcmk__release_node_capacity(GHashTable *current_utilization,
	const pcmk_resource_t *rsc)
	{
	struct calculate_data data = {
	.current_utilization = current_utilization,
	.plus = true,
	};

	g_hash_table_foreach(rsc->utilization, update_utilization_value, &data);
	}


	/*
	* Functions for checking for sufficient node capacity
	*/

	struct capacity_data {
	const pcmk_node_t *node;
	const char *rsc_id;
	bool is_enough;
	};

	/*!
	* \internal
	* \brief Check whether a single utilization attribute has sufficient capacity
	*
	* \param[in] key Name of utilization attribute to check
	* \param[in] value Amount of utilization required
	* \param[in,out] user_data Capacity data (as struct capacity_data *)
	*/
	static void
	check_capacity(gpointer key, gpointer value, gpointer user_data)
	{
	int required = 0;
	int remaining = 0;
	const char *node_value_s = NULL;
	struct capacity_data *data = user_data;

	node_value_s = g_hash_table_lookup(data->node->details->utilization, key);

	required = utilization_value(value);
	remaining = utilization_value(node_value_s);

	if (required > remaining) {
	crm_debug("Remaining capacity for %s on %s (%d) is insufficient "
	"for resource %s usage (%d)",
	(const char *) key, pe__node_name(data->node), remaining,
	data->rsc_id, required);
	data->is_enough = false;
	}
	}

	/*!
	* \internal
	* \brief Check whether a node has sufficient capacity for a resource
	*
	* \param[in] node Node to check
	* \param[in] rsc_id ID of resource to check (for debug logs only)
	* \param[in] utilization Required utilization amounts
	*
	* \return true if node has sufficient capacity for resource, otherwise false
	*/
	static bool
	have_enough_capacity(const pcmk_node_t node, const char rsc_id,
	GHashTable *utilization)
	{
	struct capacity_data data = {
	.node = node,
	.rsc_id = rsc_id,
	.is_enough = true,
	};

	g_hash_table_foreach(utilization, check_capacity, &data);
	return data.is_enough;
	}

	/*!
	* \internal
	* \brief Sum the utilization requirements of a list of resources
	*
	* \param[in] orig_rsc Resource being assigned (for logging purposes)
	* \param[in] rscs Resources whose utilization should be summed
	*
	* \return Newly allocated hash table with sum of all utilization values
	* \note It is the caller's responsibility to free the return value using
	* g_hash_table_destroy().
	*/
	static GHashTable *
	sum_resource_utilization(const pcmk_resource_t orig_rsc, GList rscs)
	{
	GHashTable *utilization = pcmk__strkey_table(free, free);

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->add_utilization(rsc, orig_rsc, rscs, utilization);
	}
	return utilization;
	}

	/*!
	* \internal
	* \brief Ban resource from nodes with insufficient utilization capacity
	*
	* \param[in,out] rsc Resource to check
	*
	* \return Allowed node for \p rsc with most spare capacity, if there are no
	* nodes with enough capacity for \p rsc and all its colocated resources
	*/
	const pcmk_node_t *
	pcmk__ban_insufficient_capacity(pcmk_resource_t *rsc)
	{
	bool any_capable = false;
	char *rscs_id = NULL;
	pcmk_node_t *node = NULL;
	const pcmk_node_t *most_capable_node = NULL;
	GList *colocated_rscs = NULL;
	GHashTable *unassigned_utilization = NULL;
	GHashTableIter iter;

	CRM_CHECK(rsc != NULL, return NULL);

	// The default placement strategy ignores utilization
	if (pcmk__str_eq(rsc->cluster->placement_strategy, "default",
	pcmk__str_casei)) {
	return NULL;
	}

	// Check whether any resources are colocated with this one
	colocated_rscs = rsc->cmds->colocated_resources(rsc, NULL, NULL);
	if (colocated_rscs == NULL) {
	return NULL;
	}

	rscs_id = crm_strdup_printf("%s and its colocated resources", rsc->id);

	// If rsc isn't in the list, add it so we include its utilization
	if (g_list_find(colocated_rscs, rsc) == NULL) {
	colocated_rscs = g_list_append(colocated_rscs, rsc);
	}

	// Sum utilization of colocated resources that haven't been assigned yet
	unassigned_utilization = sum_resource_utilization(rsc, colocated_rscs);

	// Check whether any node has enough capacity for all the resources
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, true, false)) {
	continue;
	}

	if (have_enough_capacity(node, rscs_id, unassigned_utilization)) {
	any_capable = true;
	}

	// Keep track of node with most free capacity
	if ((most_capable_node == NULL)
	\|\| (pcmk__compare_node_capacities(node, most_capable_node) < 0)) {
	most_capable_node = node;
	}
	}

	if (any_capable) {
	// If so, ban resource from any node with insufficient capacity
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)
	&& !have_enough_capacity(node, rscs_id,
	unassigned_utilization)) {
	pcmk__rsc_debug(rsc, "%s does not have enough capacity for %s",
	pe__node_name(node), rscs_id);
	resource_location(rsc, node, -INFINITY, "__limit_utilization__",
	rsc->cluster);
	}
	}
	most_capable_node = NULL;

	} else {
	// Otherwise, ban from nodes with insufficient capacity for rsc alone
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)
	&& !have_enough_capacity(node, rsc->id, rsc->utilization)) {
	pcmk__rsc_debug(rsc, "%s does not have enough capacity for %s",
	pe__node_name(node), rsc->id);
	resource_location(rsc, node, -INFINITY, "__limit_utilization__",
	rsc->cluster);
	}
	}
	}

	g_hash_table_destroy(unassigned_utilization);
	g_list_free(colocated_rscs);
	free(rscs_id);

	pe__show_node_scores(true, rsc, "Post-utilization", rsc->allowed_nodes,
	rsc->cluster);
	return most_capable_node;
	}

	/*!
	* \internal
	* \brief Create a new load_stopped pseudo-op for a node
	*
	* \param[in,out] node Node to create op for
	*
	* \return Newly created load_stopped op
	*/
	static pcmk_action_t *
	new_load_stopped_op(pcmk_node_t *node)
	{
	char *load_stopped_task = crm_strdup_printf(PCMK_ACTION_LOAD_STOPPED "_%s",
	node->details->uname);
	pcmk_action_t *load_stopped = get_pseudo_op(load_stopped_task,
	node->details->data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(node);
	pe__clear_action_flags(load_stopped, pcmk_action_optional);
	}
	free(load_stopped_task);
	return load_stopped;
	}

	/*!
	* \internal
	* \brief Create utilization-related internal constraints for a resource
	*
	* \param[in,out] rsc Resource to create constraints for
	* \param[in] allowed_nodes List of allowed next nodes for \p rsc
	*/
	void
	pcmk__create_utilization_constraints(pcmk_resource_t *rsc,
	const GList *allowed_nodes)
	{
	const GList *iter = NULL;
	pcmk_action_t *load_stopped = NULL;

	pcmk__rsc_trace(rsc,
	"Creating utilization constraints for %s - strategy: %s",
	rsc->id, rsc->cluster->placement_strategy);

	// "stop rsc then load_stopped" constraints for current nodes
	for (iter = rsc->running_on; iter != NULL; iter = iter->next) {
	load_stopped = new_load_stopped_op(iter->data);
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL, NULL, load_stopped,
	pcmk__ar_if_on_same_node_or_target, rsc->cluster);
	}

	// "load_stopped then start/migrate_to rsc" constraints for allowed nodes
	for (iter = allowed_nodes; iter; iter = iter->next) {
	load_stopped = new_load_stopped_op(iter->data);
	pcmk__new_ordering(NULL, NULL, load_stopped, rsc, start_key(rsc), NULL,
	pcmk__ar_if_on_same_node_or_target, rsc->cluster);
	pcmk__new_ordering(NULL, NULL, load_stopped,
	rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_TO, 0),
	NULL,
	pcmk__ar_if_on_same_node_or_target, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Output node capacities if enabled
	*
	* \param[in] desc Prefix for output
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__show_node_capacities(const char desc, pcmk_scheduler_t scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	return;
	}
	for (const GList *iter = scheduler->nodes;
	iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;
	pcmk__output_t *out = scheduler->priv;

	out->message(out, "node-capacity", node, desc);
	}
	}
	diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
	index ceb8348029..1b81b669cb 100644
	--- a/lib/pacemaker/pcmk_scheduler.c
	+++ b/lib/pacemaker/pcmk_scheduler.c
	@@ -1,820 +1,820 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	/*!
	* \internal
	* \brief Do deferred action checks after assignment
	*
	* When unpacking the resource history, the scheduler checks for resource
	* configurations that have changed since an action was run. However, at that
	* time, bundles using the REMOTE_CONTAINER_HACK don't have their final
	* parameter information, so instead they add a deferred check to a list. This
	* function processes one entry in that list.
	*
	* \param[in,out] rsc Resource that action history is for
	* \param[in,out] node Node that action history is for
	* \param[in] rsc_op Action history entry
	* \param[in] check Type of deferred check to do
	*/
	static void
	check_params(pcmk_resource_t rsc, pcmk_node_t node, const xmlNode *rsc_op,
	enum pcmk__check_parameters check)
	{
	const char *reason = NULL;
	pcmk__op_digest_t *digest_data = NULL;

	switch (check) {
	case pcmk__check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL)) {
	reason = "action definition changed";
	}
	break;

	case pcmk__check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s has "
	"no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}
	if (reason != NULL) {
	pe__clear_failcount(rsc, node, reason, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has failcount clearing scheduled on a node
	*
	* \param[in] node Node to check
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc has failcount clearing scheduled on \p node,
	* otherwise false
	*/
	static bool
	failcount_clear_action_exists(const pcmk_node_t *node,
	const pcmk_resource_t *rsc)
	{
	GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
	TRUE);

	if (list != NULL) {
	g_list_free(list);
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it reached its failure threshold there
	*
	* \param[in,out] data Resource to check failure threshold for
	* \param[in] user_data Node to check resource on
	*/
	static void
	check_failure_threshold(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, check_failure_threshold, user_data);
	return;
	}

	if (!failcount_clear_action_exists(node, rsc)) {
	/* Don't force the resource away from this node due to a failcount
	* that's going to be cleared.
	*
	* @TODO Failcount clearing can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* schedule_resource_actions() via check_params(). This runs well before
	* then, so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't. Worst case, we stop or move the
	* resource, then move it back in the next transition.
	*/
	pcmk_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -INFINITY, "__fail_limit__",
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief If resource has exclusive discovery, ban node if not allowed
	*
	* Location constraints have a resource-discovery option that allows users to
	* specify where probes are done for the affected resource. If this is set to
	* exclusive, probes will only be done on nodes listed in exclusive constraints.
	* This function bans the resource from the node if the node is not listed.
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Node to check resource on
	*/
	static void
	apply_exclusive_discovery(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	if (rsc->exclusive_discover
	\|\| pe__const_top_resource(rsc, false)->exclusive_discover) {
	pcmk_node_t *match = NULL;

	// If this is a collective resource, apply recursively to children
	g_list_foreach(rsc->children, apply_exclusive_discovery, user_data);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if ((match != NULL)
	&& (match->rsc_discover_mode != pcmk_probe_exclusive)) {
	match->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Apply stickiness to a resource if appropriate
	*
	* \param[in,out] data Resource to check for stickiness
	* \param[in] user_data Ignored
	*/
	static void
	apply_stickiness(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	pcmk_node_t *node = NULL;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, apply_stickiness, NULL);
	return;
	}

	/* A resource is sticky if it is managed, has stickiness configured, and is
	* active on a single node.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| (rsc->stickiness < 1) \|\| !pcmk__list_of_1(rsc->running_on)) {
	return;
	}

	node = rsc->running_on->data;

	/* In a symmetric cluster, stickiness can always be used. In an
	* asymmetric cluster, we have to check whether the resource is still
	* allowed on the node, so we don't keep the resource somewhere it is no
	* longer explicitly enabled.
	*/
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id) == NULL)) {
	pcmk__rsc_debug(rsc,
	"Ignoring %s stickiness because the cluster is "
	"asymmetric and %s is not explicitly allowed",
	rsc->id, pe__node_name(node));
	return;
	}

	pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
	rsc->id, rsc->stickiness, pe__node_name(node));
	resource_location(rsc, node, rsc->stickiness, "stickiness", rsc->cluster);
	}

	/*!
	* \internal
	* \brief Apply shutdown locks for all resources as appropriate
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	apply_shutdown_locks(pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	return;
	}
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->shutdown_lock(rsc);
	}
	}

	/*!
	* \internal
	* \brief Calculate the number of available nodes in the cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	count_available_nodes(pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(scheduler->flags, pcmk_sched_no_compat)) {
	return;
	}

	// @COMPAT for API backward compatibility only (cluster does not use value)
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if ((node != NULL) && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	scheduler->max_valid_nodes++;
	}
	}
	crm_trace("Online node count: %d", scheduler->max_valid_nodes);
	}

	/*
	* \internal
	* \brief Apply node-specific scheduling criteria
	*
	* After the CIB has been unpacked, process node-specific scheduling criteria
	* including shutdown locks, location constraints, resource stickiness,
	* migration thresholds, and exclusive resource discovery.
	*/
	static void
	apply_node_criteria(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Applying node-specific scheduling criteria");
	apply_shutdown_locks(scheduler);
	count_available_nodes(scheduler);
	pcmk__apply_locations(scheduler);
	g_list_foreach(scheduler->resources, apply_stickiness, NULL);

	for (GList *node_iter = scheduler->nodes; node_iter != NULL;
	node_iter = node_iter->next) {
	for (GList *rsc_iter = scheduler->resources; rsc_iter != NULL;
	rsc_iter = rsc_iter->next) {
	check_failure_threshold(rsc_iter->data, node_iter->data);
	apply_exclusive_discovery(rsc_iter->data, node_iter->data);
	}
	}
	}

	/*!
	* \internal
	* \brief Assign resources to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	assign_resources(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;

	crm_trace("Assigning resources to nodes");

	if (!pcmk__str_eq(scheduler->placement_strategy, "default",
	pcmk__str_casei)) {
	pcmk__sort_resources(scheduler);
	}
	pcmk__show_node_capacities("Original", scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	/* Assign remote connection resources first (which will also assign any
	* colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
	rsc->id);
	rsc->cmds->assign(rsc, rsc->partial_migration_target, true);
	}
	}
	}

	/* now do the rest of the resources */
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (!rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
	rsc->xml->name, rsc->id);
	rsc->cmds->assign(rsc, NULL, true);
	}
	}

	pcmk__show_node_capacities("Remaining", scheduler);
	}

	/*!
	* \internal
	* \brief Schedule fail count clearing on online nodes if resource is orphaned
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Ignored
	*/
	static void
	clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	return;
	}
	crm_trace("Clear fail counts for orphaned resource %s", rsc->id);

	/* There's no need to recurse into rsc->children because those
	* should just be unassigned clone instances.
	*/

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *clear_op = NULL;

	if (!node->details->online) {
	continue;
	}
	if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
	continue;
	}

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	rsc->cluster);

	/* We can't use order_action_then_stop() here because its
	* pcmk__ar_guest_allowed breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
	NULL, pcmk__ar_ordered, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Schedule any resource actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_resource_actions(pcmk_scheduler_t *scheduler)
	{
	// Process deferred action checks
	pe__foreach_param_check(scheduler, check_params);
	pe__free_param_checks(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_trace("Scheduling probes");
	pcmk__schedule_probes(scheduler);
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	g_list_foreach(scheduler->resources, clear_failcounts_if_orphaned,
	NULL);
	}

	crm_trace("Scheduling resource actions");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->create_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its descendants are managed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if resource or any descendant is managed, otherwise false
	*/
	static bool
	is_managed(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (is_managed((pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether any resources in the cluster are managed
	*
	* \param[in] scheduler Scheduler data
	*
	* \return true if any resource is managed, otherwise false
	*/
	static bool
	any_managed_resources(const pcmk_scheduler_t *scheduler)
	{
	for (const GList *iter = scheduler->resources;
	iter != NULL; iter = iter->next) {
	if (is_managed((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node requires fencing
	*
	* \param[in] node Node to check
	* \param[in] have_managed Whether any resource in cluster is managed
	*
	* \return true if \p node should be fenced, otherwise false
	*/
	static bool
	needs_fencing(const pcmk_node_t *node, bool have_managed)
	{
	return have_managed && node->details->unclean
	&& pe_can_fence(node->details->data_set, node);
	}

	/*!
	* \internal
	* \brief Check whether a node requires shutdown
	*
	* \param[in] node Node to check
	*
	* \return true if \p node should be shut down, otherwise false
	*/
	static bool
	needs_shutdown(const pcmk_node_t *node)
	{
	if (pe__is_guest_or_remote_node(node)) {
	/* Do not send shutdown actions for Pacemaker Remote nodes.
	* @TODO We might come up with a good use for this in the future.
	*/
	return false;
	}
	return node->details->online && node->details->shutdown;
	}

	/*!
	* \internal
	* \brief Track and order non-DC fencing
	*
	* \param[in,out] list List of existing non-DC fencing actions
	* \param[in,out] action Fencing action to prepend to \p list
	* \param[in] scheduler Scheduler data
	*
	* \return (Possibly new) head of \p list
	*/
	static GList *
	add_nondc_fencing(GList list, pcmk_action_t action,
	const pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)
	&& (list != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
	}
	return g_list_prepend(list, action);
	}

	/*!
	* \internal
	* \brief Schedule a node for fencing
	*
	* \param[in,out] node Node that requires fencing
	*/
	static pcmk_action_t *
	schedule_fencing(pcmk_node_t *node)
	{
	pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
	FALSE, node->details->data_set);

	- pe_warn("Scheduling node %s for fencing", pe__node_name(node));
	+ pcmk__config_warn("Scheduling node %s for fencing", pe__node_name(node));
	pcmk__order_vs_fence(fencing, node->details->data_set);
	return fencing;
	}

	/*!
	* \internal
	* \brief Create and order node fencing and shutdown actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *dc_down = NULL;
	bool integrity_lost = false;
	bool have_managed = any_managed_resources(scheduler);
	GList *fencing_ops = NULL;
	GList *shutdown_ops = NULL;

	crm_trace("Scheduling fencing and shutdowns as needed");
	if (!have_managed) {
	crm_notice("No fencing will be done until there are resources "
	"to manage");
	}

	// Check each node for whether it needs fencing or shutdown
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *fencing = NULL;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && have_managed
	&& pe_can_fence(scheduler, node)) {
	pcmk__fence_guest(node);
	}
	continue;
	}

	if (needs_fencing(node, have_managed)) {
	fencing = schedule_fencing(node);

	// Track DC and non-DC fence actions separately
	if (node->details->is_dc) {
	dc_down = fencing;
	} else {
	fencing_ops = add_nondc_fencing(fencing_ops, fencing,
	scheduler);
	}

	} else if (needs_shutdown(node)) {
	pcmk_action_t *down_op = pcmk__new_shutdown_action(node);

	// Track DC and non-DC shutdown actions separately
	if (node->details->is_dc) {
	dc_down = down_op;
	} else {
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if ((fencing == NULL) && node->details->unclean) {
	integrity_lost = true;
	- pe_warn("Node %s is unclean but cannot be fenced",
	- pe__node_name(node));
	+ pcmk__config_warn("Node %s is unclean but cannot be fenced",
	+ pe__node_name(node));
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	- pe_warn("Resource functionality and data integrity cannot be "
	- "guaranteed (configure, enable, and test fencing to "
	- "correct this)");
	+ pcmk__config_warn("Resource functionality and data integrity "
	+ "cannot be guaranteed (configure, enable, "
	+ "and test fencing to correct this)");

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	crm_notice("Unclean nodes will not be fenced until quorum is "
	"attained or no-quorum-policy is set to ignore");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	pcmk__order_after_each(dc_down, shutdown_ops);
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	pcmk__order_after_each(dc_down, fencing_ops);
	} else if (fencing_ops != NULL) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
	pcmk__ar_ordered);
	}
	}
	g_list_free(fencing_ops);
	g_list_free(shutdown_ops);
	}

	static void
	log_resource_details(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;

	/* Due to the `crm_mon --node=` feature, out->message() for all the
	* resource-related messages expects a list of nodes that we are allowed to
	* output information for. Here, we create a wildcard to match all nodes.
	*/
	all = g_list_prepend(all, (gpointer) "*");

	for (GList *item = scheduler->resources; item != NULL; item = item->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) item->data;

	// Log all resources except inactive orphans
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role != pcmk_role_stopped)) {
	out->message(out, crm_map_element_name(rsc->xml), 0, rsc, all, all);
	}
	}

	g_list_free(all);
	}

	static void
	log_all_actions(pcmk_scheduler_t *scheduler)
	{
	/* This only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	pcmk__output_t *prev_out = scheduler->priv;
	pcmk__output_t *out = NULL;

	if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
	return;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	pcmk__output_set_log_level(out, LOG_NOTICE);
	scheduler->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	scheduler->priv = prev_out;
	}

	/*!
	* \internal
	* \brief Log all required but unrunnable actions at trace level
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
	{
	const uint64_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	crm_trace("Required but unrunnable actions:");
	for (const GList *iter = scheduler->actions;
	iter != NULL; iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (!pcmk_any_flags_set(action->flags, flags)) {
	pcmk__log_action("\t", action, true);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack the CIB for scheduling
	*
	* \param[in,out] cib CIB XML to unpack (may be NULL if already unpacked)
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_cib(xmlNode cib, unsigned long long flags, pcmk_scheduler_t scheduler)
	{
	const char* localhost_save = NULL;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_status)) {
	crm_trace("Reusing previously calculated cluster status");
	pe__set_working_set_flags(scheduler, flags);
	return;
	}

	if (scheduler->localhost) {
	localhost_save = scheduler->localhost;
	}

	CRM_ASSERT(cib != NULL);
	crm_trace("Calculating cluster status");

	/* This will zero the entire struct without freeing anything first, so
	* callers should never call pcmk__schedule_actions() with a populated data
	* set unless pcmk_sched_have_status is set (i.e. cluster_status() was
	* previously called, whether directly or via pcmk__schedule_actions()).
	*/
	set_working_set_defaults(scheduler);

	if (localhost_save) {
	scheduler->localhost = localhost_save;
	}

	pe__set_working_set_flags(scheduler, flags);
	scheduler->input = cib;
	cluster_status(scheduler); // Sets pcmk_sched_have_status
	}

	/*!
	* \internal
	* \brief Run the scheduler for a given CIB
	*
	* \param[in,out] cib CIB XML to use as scheduler input
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
	pcmk_scheduler_t *scheduler)
	{
	unpack_cib(cib, flags, scheduler);
	pcmk__set_assignment_methods(scheduler);
	pcmk__apply_node_health(scheduler);
	pcmk__unpack_constraints(scheduler);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_validate_only)) {
	return;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)
	&& pcmk__is_daemon) {
	log_resource_details(scheduler);
	}

	apply_node_criteria(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	return;
	}

	pcmk__create_internal_constraints(scheduler);
	pcmk__handle_rsc_config_changes(scheduler);
	assign_resources(scheduler);
	schedule_resource_actions(scheduler);

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(scheduler);

	schedule_fencing_and_shutdowns(scheduler);
	pcmk__apply_orderings(scheduler);
	log_all_actions(scheduler);
	pcmk__create_graph(scheduler);

	if (get_crm_log_level() == LOG_TRACE) {
	log_unrunnable_actions(scheduler);
	}
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 4f076eea42..d2d9803d44 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2224 +1,2226 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pe__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	crm_xml_add(rsc, XML_ATTR_ID, name);
	crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
	crm_xml_add(rsc, XML_ATTR_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	- pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
	+ pcmk__config_err("Specifying the 'control-port' for %s requires "
	+ "'replicas-per-host=1'", data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = create_xml_node(xml_ip, "operations");
	crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network, "host",
	pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = create_xml_node(xml_container, "operations");
	crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pe__set_resource_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = ID(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, "remote", "-INFINITY",
	parent->cluster);
	} else {
	node->weight = -INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pe__is_guest_or_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pe__is_guest_node() functions correctly immediately
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	g_hash_table_insert(replica->node->details->attrs,
	strdup(CRM_ATTR_KIND), strdup("container"));

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pe__set_resource_flags(replica->remote,
	pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));

	CRM_ASSERT(mount != NULL);
	mount->source = strdup(source);
	mount->target = strdup(target);
	pcmk__str_update(&mount->options, options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	xmlNode xml, const char field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pe__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pe__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", ID(mount_xml), flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid promoted-max
	value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, "masters");
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	// Default replicas to promoted-max if it was specified and 1 otherwise
	value = crm_element_value(xml_obj, "replicas");
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, "replicas-per-host");
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
	bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
	bundle_data->image = crm_element_value_copy(xml_obj, "image");
	bundle_data->container_network = crm_element_value_copy(xml_obj, "network");

	xml_obj = first_named_child(rsc->xml, "network");
	if(xml_obj) {

	bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
	bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
	bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
	bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
	value = crm_element_value(xml_obj, "add-host");
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
	port->source = crm_element_value_copy(xml_child, "port");

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, "range");
	} else {
	port->target = crm_element_value_copy(xml_child, "internal-port");
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	- pe_err("Invalid port directive %s", ID(xml_child));
	+ pcmk__config_err("Invalid port directive %s", ID(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = first_named_child(rsc->xml, "storage");
	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	const char *source = crm_element_value(xml_child, "source-dir");
	const char *target = crm_element_value(xml_child, "target-dir");
	const char *options = crm_element_value(xml_child, "options");
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, "source-dir-root");
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	- pe_err("Invalid mount directive %s", ID(xml_child));
	+ pcmk__config_err("Invalid mount directive %s", ID(xml_child));
	}
	}

	xml_obj = first_named_child(rsc->xml, "primitive");
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
	add_node_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	- pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
	- rsc->id, ID(xml_obj));
	+ pcmk__config_err("Cannot control %s inside %s without either "
	+ "ip-range-start or control-port",
	+ rsc->id, ID(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = calloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting control-port.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pe__set_resource_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
	XML_RSC_ATTR_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	g_hash_table_replace(bundle_data->child->meta,
	strdup(XML_RSC_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	- pe_err("Failed unpacking resource %s", rsc->id);
	+ pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (replica->node->details == node->details) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " XML_ATTR_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
	"id", rsc->id,
	"type", container_agent_str(bundle_data->agent_type),
	"image", bundle_data->image,
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	"maintenance",
	pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"description", desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pcmk__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pcmk_node_t *node = NULL;
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pe__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk_rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
	index fc861a94c1..381ba1e248 100644
	--- a/lib/pengine/clone.c
	+++ b/lib/pengine/clone.c
	@@ -1,1518 +1,1518 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED_LEGACY "s"
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED_LEGACY "s"
	#else
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED
	#endif

	typedef struct clone_variant_data_s {
	int clone_max;
	int clone_node_max;

	int promoted_max;
	int promoted_node_max;

	int total_clones;

	uint32_t flags; // Group of enum pcmk__clone_flags

	notify_data_t *stop_notify;
	notify_data_t *start_notify;
	notify_data_t *demote_notify;
	notify_data_t *promote_notify;

	xmlNode *xml_obj_child;
	} clone_variant_data_t;

	#define get_clone_variant_data(data, rsc) \
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_clone)); \
	data = (clone_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be run
	*
	* \param[in] clone Clone or clone instance to check
	*
	* \return Maximum instances for \p clone
	*/
	int
	pe__clone_max(const pcmk_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed per node
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum allowed instances per node for \p clone
	*/
	int
	pe__clone_node_max(const pcmk_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_node_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_max(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_node_max(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_node_max;
	}

	static GList *
	sorted_hash_table_values(GHashTable *table)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
	retval = g_list_prepend(retval, (char *) value);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) strcmp);
	return retval;
	}

	static GList *
	nodes_with_status(GHashTable table, const char status)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!strcmp((char *) value, status)) {
	retval = g_list_prepend(retval, key);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
	return retval;
	}

	static GString *
	node_list_to_str(const GList *list)
	{
	GString *retval = NULL;

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__add_word(&retval, 1024, (const char *) iter->data);
	}

	return retval;
	}

	static void
	clone_header(pcmk__output_t out, int rc, const pcmk_resource_t *rsc,
	clone_variant_data_t clone_data, const char desc)
	{
	GString *attrs = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__add_separated_word(&attrs, 64, "promotable", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	pcmk__add_separated_word(&attrs, 64, "unique", ", ");
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
	}

	if (attrs != NULL) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s] (%s)%s%s%s",
	rsc->id, ID(clone_data->xml_obj_child),
	(const char *) attrs->str, desc ? " (" : "",
	desc ? desc : "", desc ? ")" : "");
	g_string_free(attrs, TRUE);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s]%s%s%s",
	rsc->id, ID(clone_data->xml_obj_child),
	desc ? " (" : "", desc ? desc : "",
	desc ? ")" : "");
	}
	}

	void
	pe__force_anon(const char standard, pcmk_resource_t rsc, const char *rid,
	pcmk_scheduler_t *scheduler)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	- pe_warn("Ignoring " XML_RSC_ATTR_UNIQUE " for %s because %s resources "
	- "such as %s can be used only as anonymous clones",
	- rsc->id, standard, rid);
	+ pcmk__config_warn("Ignoring " XML_RSC_ATTR_UNIQUE " for %s because "
	+ "%s resources such as %s can be used only as "
	+ "anonymous clones", rsc->id, standard, rid);

	clone_data->clone_node_max = 1;
	clone_data->clone_max = QB_MIN(clone_data->clone_max,
	g_list_length(scheduler->nodes));
	}
	}

	pcmk_resource_t *
	find_clone_instance(const pcmk_resource_t rsc, const char sub_id)
	{
	char *child_id = NULL;
	pcmk_resource_t *child = NULL;
	const char *child_base = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	child_base = ID(clone_data->xml_obj_child);
	child_id = crm_strdup_printf("%s:%s", child_base, sub_id);
	child = pe_find_resource(rsc->children, child_id);

	free(child_id);
	return child;
	}

	pcmk_resource_t *
	pe__create_clone_child(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	gboolean as_orphan = FALSE;
	char *inc_num = NULL;
	char *inc_max = NULL;
	pcmk_resource_t *child_rsc = NULL;
	xmlNode *child_copy = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);

	if (clone_data->total_clones >= clone_data->clone_max) {
	// If we've already used all available instances, this is an orphan
	as_orphan = TRUE;
	}

	// Allocate instance numbers in numerical order (starting at 0)
	inc_num = pcmk__itoa(clone_data->total_clones);
	inc_max = pcmk__itoa(clone_data->clone_max);

	child_copy = copy_xml(clone_data->xml_obj_child);

	crm_xml_add(child_copy, XML_RSC_ATTR_INCARNATION, inc_num);

	if (pe__unpack_resource(child_copy, &child_rsc, rsc,
	scheduler) != pcmk_rc_ok) {
	goto bail;
	}
	/* child_rsc->globally_unique = rsc->globally_unique; */

	CRM_ASSERT(child_rsc);
	clone_data->total_clones += 1;
	pcmk__rsc_trace(child_rsc, "Setting clone attributes for: %s",
	child_rsc->id);
	rsc->children = g_list_append(rsc->children, child_rsc);
	if (as_orphan) {
	pe__set_resource_flags_recursive(child_rsc, pcmk_rsc_removed);
	}

	add_hash_param(child_rsc->meta, PCMK_META_CLONE_MAX, inc_max);
	pcmk__rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);

	bail:
	free(inc_num);
	free(inc_max);

	return child_rsc;
	}

	/*!
	* \internal
	* \brief Unpack a nonnegative integer value from a resource meta-attribute
	*
	* \param[in] rsc Resource with meta-attribute
	* \param[in] meta_name Name of meta-attribute to unpack
	* \param[in] deprecated_name If not NULL, try unpacking this
	* if \p meta_name is unset
	* \param[in] default_value Value to use if unset
	*
	* \return Integer parsed from resource's specified meta-attribute if a valid
	* nonnegative integer, \p default_value if unset, or 0 if invalid
	*/
	static int
	unpack_meta_int(const pcmk_resource_t rsc, const char meta_name,
	const char *deprecated_name, int default_value)
	{
	int integer = default_value;
	const char *value = g_hash_table_lookup(rsc->meta, meta_name);

	if ((value == NULL) && (deprecated_name != NULL)) {
	value = g_hash_table_lookup(rsc->meta, deprecated_name);
	}
	if (value != NULL) {
	pcmk__scan_min_int(value, &integer, 0);
	}
	return integer;
	}

	gboolean
	clone_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	int lpc = 0;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = rsc->xml;
	clone_variant_data_t *clone_data = NULL;

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	clone_data = calloc(1, sizeof(clone_variant_data_t));
	rsc->variant_opaque = clone_data;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	// Use 1 as default but 0 for minimum and invalid
	// @COMPAT PCMK_XA_PROMOTED_MAX_LEGACY deprecated since 2.0.0
	clone_data->promoted_max = unpack_meta_int(rsc, PCMK_META_PROMOTED_MAX,
	PCMK_XA_PROMOTED_MAX_LEGACY,
	1);

	// Use 1 as default but 0 for minimum and invalid
	// @COMPAT PCMK_XA_PROMOTED_NODE_MAX_LEGACY deprecated since 2.0.0
	clone_data->promoted_node_max =
	unpack_meta_int(rsc, PCMK_META_PROMOTED_NODE_MAX,
	PCMK_XA_PROMOTED_NODE_MAX_LEGACY, 1);
	}

	// Implied by calloc()
	/* clone_data->xml_obj_child = NULL; */

	// Use 1 as default but 0 for minimum and invalid
	clone_data->clone_node_max = unpack_meta_int(rsc, PCMK_META_CLONE_NODE_MAX,
	NULL, 1);

	/* Use number of nodes (but always at least 1, which is handy for crm_verify
	* for a CIB without nodes) as default, but 0 for minimum and invalid
	*/
	clone_data->clone_max = unpack_meta_int(rsc, PCMK_META_CLONE_MAX, NULL,
	QB_MAX(1, g_list_length(scheduler->nodes)));

	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_ORDERED))) {
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", rsc->id,
	clone_data->flags,
	pcmk__clone_ordered,
	"pcmk__clone_ordered");
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_node_max > 1)) {

	pcmk__config_err("Ignoring " PCMK_META_CLONE_NODE_MAX " of %d for %s "
	"because anonymous clones support only one instance "
	"per node", clone_data->clone_node_max, rsc->id);
	clone_data->clone_node_max = 1;
	}

	pcmk__rsc_trace(rsc, "Options for %s", rsc->id);
	pcmk__rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
	pcmk__rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
	pcmk__rsc_trace(rsc, "\tClone is unique: %s",
	pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	pcmk__rsc_trace(rsc, "\tClone is promotable: %s",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable));

	// Clones may contain a single group or primitive
	for (a_child = pcmk__xe_first_child(xml_obj); a_child != NULL;
	a_child = pcmk__xe_next(a_child)) {

	if (pcmk__str_any_of((const char *)a_child->name, XML_CIB_TAG_RESOURCE, XML_CIB_TAG_GROUP, NULL)) {
	clone_data->xml_obj_child = a_child;
	break;
	}
	}

	if (clone_data->xml_obj_child == NULL) {
	pcmk__config_err("%s has nothing to clone", rsc->id);
	return FALSE;
	}

	/*
	* Make clones ever so slightly sticky by default
	*
	* This helps ensure clone instances are not shuffled around the cluster
	* for no benefit in situations when pre-allocation is not appropriate
	*/
	if (g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_STICKINESS) == NULL) {
	add_hash_param(rsc->meta, XML_RSC_ATTR_STICKINESS, "1");
	}

	/* This ensures that the globally-unique value always exists for children to
	* inherit when being unpacked, as well as in resource agents' environment.
	*/
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE,
	pe__rsc_bool_str(rsc, pcmk_rsc_unique));

	if (clone_data->clone_max <= 0) {
	/* Create one child instance so that unpack_find_resource() will hook up
	* any orphans up to the parent correctly.
	*/
	if (pe__create_clone_child(rsc, scheduler) == NULL) {
	return FALSE;
	}

	} else {
	// Create a child instance for each available instance number
	for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
	if (pe__create_clone_child(rsc, scheduler) == NULL) {
	return FALSE;
	}
	}
	}

	pcmk__rsc_trace(rsc, "Added %d children to resource %s...",
	clone_data->clone_max, rsc->id);
	return TRUE;
	}

	gboolean
	clone_active(pcmk_resource_t * rsc, gboolean all)
	{
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean child_active = child_rsc->fns->active(child_rsc, all);

	if (all == FALSE && child_active) {
	return TRUE;
	} else if (all && child_active == FALSE) {
	return FALSE;
	}
	}

	if (all) {
	return TRUE;
	} else {
	return FALSE;
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	short_print(const char list, const char prefix, const char *type,
	const char suffix, long options, void print_data)
	{
	if(suffix == NULL) {
	suffix = "";
	}

	if (!pcmk__str_empty(list)) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	status_print("%s%s: [ %s ]%s", prefix, type, list, suffix);

	if (options & pe_print_html) {
	status_print("</li>\n");

	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}

	}
	}

	static const char *
	configured_role_str(pcmk_resource_t * rsc)
	{
	const char *target_role = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_TARGET_ROLE);

	if ((target_role == NULL) && rsc->children && rsc->children->data) {
	pcmk_resource_t *instance = rsc->children->data; // Any instance will do

	target_role = g_hash_table_lookup(instance->meta,
	XML_RSC_ATTR_TARGET_ROLE);
	}
	return target_role;
	}

	static enum rsc_role_e
	configured_role(pcmk_resource_t *rsc)
	{
	const char *target_role = configured_role_str(rsc);

	if (target_role) {
	return text2role(target_role);
	}
	return pcmk_role_unknown;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	clone_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	char *child_text = crm_strdup_printf("%s ", pre_text);
	const char *target_role = configured_role_str(rsc);
	GList *gIter = rsc->children;

	status_print("%s<clone ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc->id);
	status_print("multi_state=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable));
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</clone>\n", pre_text);
	free(child_text);
	}

	bool
	is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any)
	{
	GList *gIter;
	bool all = !any;

	if (pcmk_is_set(rsc->flags, flag)) {
	if(any) {
	return TRUE;
	}
	} else if(all) {
	return FALSE;
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	if(is_set_recursive(gIter->data, flag, any)) {
	if(any) {
	return TRUE;
	}

	} else if(all) {
	return FALSE;
	}
	}

	if(all) {
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	clone_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	GString *list_text = NULL;
	char *child_text = NULL;
	GString *stopped_list = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	clone_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_clone_variant_data(clone_data, rsc);

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sClone Set: %s [%s]%s%s%s",
	pre_text ? pre_text : "", rsc->id, ID(clone_data->xml_obj_child),
	pcmk_is_set(rsc->flags, pcmk_rsc_promotable)? " (promotable)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (options & pe_print_clone_details) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(options, pe_print_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(options, pe_print_clone_active)) {

	pcmk__add_word(&stopped_list, 1024, child_rsc->id);
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pcmk_node_t *location = NULL;

	location = child_rsc->fns->location(child_rsc, NULL, TRUE);
	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	short_print((const char *) list_text->str, child_text,
	PROMOTED_INSTANCES, NULL, options, print_data);
	g_string_truncate(list_text, 0);
	}
	g_list_free(promoted_list);

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES " (target-role)", NULL,
	options, print_data);
	} else {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES, NULL, options, print_data);
	}

	} else {
	short_print((const char *) list_text->str, child_text, "Started",
	NULL, options, print_data);
	}
	}

	g_list_free(started_list);

	if (!pcmk_is_set(options, pe_print_clone_active)) {
	const char *state = "Stopped";
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped list for non-unique clones */
	if (stopped_list != NULL) {
	g_string_truncate(stopped_list, 0);
	}

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pcmk_node_t node = (pcmk_node_t ) nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL) {
	pcmk__add_word(&stopped_list, 1024, node->details->uname);
	}
	}
	g_list_free(list);
	}

	if (stopped_list != NULL) {
	short_print((const char *) stopped_list->str, child_text, state,
	NULL, options, print_data);
	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (stopped_list != NULL) {
	g_string_free(stopped_list, TRUE);
	}
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__clone_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );


	const char *desc = NULL;
	GList *gIter = rsc->children;
	GList *all = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;



	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	all = g_list_prepend(all, (gpointer) "*");

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);
	rc = pe__name_and_nvpairs_xml(out, true, "clone", 10,
	"id", rsc->id,
	"multi_state",
	pe__rsc_bool_str(rsc, pcmk_rsc_promotable),
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	"maintenance",
	pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"disabled", pcmk__btoa(pe__resource_is_disabled(rsc)),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"target_role", configured_role_str(rsc),
	"description", desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out);
	}

	g_list_free(all);
	return rc;
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__clone_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GHashTable *stopped = NULL;

	GString *list_text = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	const char *desc = NULL;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_clone_variant_data(clone_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(show_opts, pcmk_show_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(show_opts, pcmk_show_clone_detail)
	&& pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	g_hash_table_insert(stopped, strdup(child_rsc->id), strdup("Stopped"));
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pcmk_node_t *location = NULL;

	location = child_rsc->fns->location(child_rsc, NULL, TRUE);
	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	GList *all = NULL;

	clone_header(out, &rc, rsc, clone_data, desc);

	/* Print every resource that's a child of this clone. */
	all = g_list_prepend(all, (gpointer) "*");
	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	g_list_free(all);
	}
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return pcmk_rc_ok;
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(promoted_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	out->list_item(out, NULL, PROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	g_string_truncate(list_text, 0);
	}

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(started_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	out->list_item(out, NULL,
	UNPROMOTED_INSTANCES " (target-role): [ %s ]",
	(const char *) list_text->str);
	} else {
	out->list_item(out, NULL, UNPROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	}

	} else {
	out->list_item(out, NULL, "Started: [ %s ]",
	(const char *) list_text->str);
	}
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped table for non-unique clones */
	if (stopped != NULL) {
	g_hash_table_destroy(stopped);
	stopped = NULL;
	}

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pcmk_node_t node = (pcmk_node_t ) nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
	pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
	const char *state = "Stopped";

	if (configured_role(rsc) == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	g_hash_table_insert(stopped, strdup(node->details->uname),
	crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
	} else {
	g_hash_table_insert(stopped, strdup(node->details->uname),
	strdup(state));
	}
	}
	}
	g_list_free(list);
	}

	if (stopped != NULL) {
	GList *list = sorted_hash_table_values(stopped);

	clone_header(out, &rc, rsc, clone_data, desc);

	for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
	const char *status = status_iter->data;
	GList *nodes = nodes_with_status(stopped, status);
	GString *nodes_str = node_list_to_str(nodes);

	if (nodes_str != NULL) {
	if (nodes_str->len > 0) {
	out->list_item(out, NULL, "%s: [ %s ]", status,
	(const char *) nodes_str->str);
	}
	g_string_free(nodes_str, TRUE);
	}

	g_list_free(nodes);
	}

	g_list_free(list);
	g_hash_table_destroy(stopped);

	/* If there are no instances of this clone (perhaps because there are no
	* nodes configured), simply output the clone header by itself. This can
	* come up in PCS testing.
	*/
	} else if (active_instances == 0) {
	clone_header(out, &rc, rsc, clone_data, desc);
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	void
	clone_free(pcmk_resource_t * rsc)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pcmk__rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	free_xml(child_rsc->xml);
	child_rsc->xml = NULL;
	/* There could be a saved unexpanded xml */
	free_xml(child_rsc->orig_xml);
	child_rsc->orig_xml = NULL;
	child_rsc->fns->free(child_rsc);
	}

	g_list_free(rsc->children);

	if (clone_data) {
	CRM_ASSERT(clone_data->demote_notify == NULL);
	CRM_ASSERT(clone_data->stop_notify == NULL);
	CRM_ASSERT(clone_data->start_notify == NULL);
	CRM_ASSERT(clone_data->promote_notify == NULL);
	}

	common_free(rsc);
	}

	enum rsc_role_e
	clone_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e clone_role = pcmk_role_unknown;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, current);

	if (a_role > clone_role) {
	clone_role = a_role;
	}
	}

	pcmk__rsc_trace(rsc, "%s role: %s", rsc->id, role2text(clone_role));
	return clone_role;
	}

	/*!
	* \internal
	* \brief Check whether a clone has an instance for every node
	*
	* \param[in] rsc Clone to check
	* \param[in] scheduler Scheduler data
	*/
	bool
	pe__is_universal_clone(const pcmk_resource_t *rsc,
	const pcmk_scheduler_t *scheduler)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	if (clone_data->clone_max == g_list_length(scheduler->nodes)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gboolean
	pe__clone_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	clone_variant_data_t *clone_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_clone_variant_data(clone_data, rsc);
	passes = pcmk__str_in_list(ID(clone_data->xml_obj_child), only_rsc, pcmk__str_star_matches);

	if (!passes) {
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t *child_rsc = NULL;

	child_rsc = (const pcmk_resource_t *) iter->data;
	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}
	}
	return !passes;
	}

	const char *
	pe__clone_child_id(const pcmk_resource_t *rsc)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);
	return ID(clone_data->xml_obj_child);
	}

	/*!
	* \internal
	* \brief Check whether a clone is ordered
	*
	* \param[in] clone Clone resource to check
	*
	* \return true if clone is ordered, otherwise false
	*/
	bool
	pe__clone_is_ordered(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	return pcmk_is_set(clone_data->flags, pcmk__clone_ordered);
	}

	/*!
	* \internal
	* \brief Set a clone flag
	*
	* \param[in,out] clone Clone resource to set flag for
	* \param[in] flag Clone flag to set
	*
	* \return Standard Pacemaker return code (either pcmk_rc_ok if flag was not
	* already set or pcmk_rc_already if it was)
	*/
	int
	pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	if (pcmk_is_set(clone_data->flags, flag)) {
	return pcmk_rc_already;
	}
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", clone->id,
	clone_data->flags, flag, "flag");
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a clone flag is set
	*
	* \param[in] group Clone resource to check
	* \param[in] flags Flag or flags to check
	*
	* \return \c true if all \p flags are set for \p clone, otherwise \c false
	*/
	bool
	pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	CRM_ASSERT(clone_data != NULL);

	return pcmk_all_flags_set(clone_data->flags, flags);
	}

	/*!
	* \internal
	* \brief Create pseudo-actions needed for promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[in] any_promoting Whether any instances will be promoted
	* \param[in] any_demoting Whether any instance will be demoted
	*/
	void
	pe__create_promotable_pseudo_ops(pcmk_resource_t *clone, bool any_promoting,
	bool any_demoting)
	{
	pcmk_action_t *action = NULL;
	pcmk_action_t *action_complete = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	// Create a "promote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTE,
	!any_promoting, true);

	// Create a "promoted" action for when all promotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTED,
	!any_promoting, true);
	action_complete->priority = INFINITY;

	// Create notification pseudo-actions for promotion
	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_PROMOTE,
	action,
	action_complete);
	}

	// Create a "demote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTE,
	!any_demoting, true);

	// Create a "demoted" action for when all demotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTED,
	!any_demoting, true);
	action_complete->priority = INFINITY;

	// Create notification pseudo-actions for demotion
	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_DEMOTE,
	action,
	action_complete);

	if (clone_data->promote_notify != NULL) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->start_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->start_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->stop_notify->pre, pcmk__ar_ordered);
	}
	}
	}

	/*!
	* \internal
	* \brief Create all notification data and actions for a clone
	*
	* \param[in,out] clone Clone to create notifications for
	*/
	void
	pe__create_clone_notifications(pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__create_action_notifications(clone, clone_data->start_notify);
	pe__create_action_notifications(clone, clone_data->stop_notify);
	pe__create_action_notifications(clone, clone_data->promote_notify);
	pe__create_action_notifications(clone, clone_data->demote_notify);
	}

	/*!
	* \internal
	* \brief Free all notification data for a clone
	*
	* \param[in,out] clone Clone to free notification data for
	*/
	void
	pe__free_clone_notification_data(pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__free_action_notification_data(clone_data->demote_notify);
	clone_data->demote_notify = NULL;

	pe__free_action_notification_data(clone_data->stop_notify);
	clone_data->stop_notify = NULL;

	pe__free_action_notification_data(clone_data->start_notify);
	clone_data->start_notify = NULL;

	pe__free_action_notification_data(clone_data->promote_notify);
	clone_data->promote_notify = NULL;
	}

	/*!
	* \internal
	* \brief Create pseudo-actions for clone start/stop notifications
	*
	* \param[in,out] clone Clone to create pseudo-actions for
	* \param[in,out] start Start action for \p clone
	* \param[in,out] stop Stop action for \p clone
	* \param[in,out] started Started action for \p clone
	* \param[in,out] stopped Stopped action for \p clone
	*/
	void
	pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
	pcmk_action_t start, pcmk_action_t started,
	pcmk_action_t stop, pcmk_action_t stopped)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	if (clone_data->start_notify == NULL) {
	clone_data->start_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_START,
	start, started);
	}

	if (clone_data->stop_notify == NULL) {
	clone_data->stop_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_STOP,
	stop, stopped);
	if ((clone_data->start_notify != NULL)
	&& (clone_data->stop_notify != NULL)) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->start_notify->pre, pcmk__ar_ordered);
	}
	}
	}

	/*!
	* \internal
	* \brief Get maximum clone resource instances per node
	*
	* \param[in] rsc Clone resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__clone_max_per_node(const pcmk_resource_t *rsc)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);
	return clone_data->clone_node_max;
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 3d0dcb60a2..42cd1dcf6b 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1195 +1,1199 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pcmk_node_t active_node(const pcmk_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	pcmk_rsc_methods_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pcmk_rsc_variant_primitive;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pcmk_rsc_variant_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pcmk_rsc_variant_bundle;
	}

	return pcmk_rsc_variant_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
	pe_rule_eval_data_t *rule_data,
	GHashTable meta_hash, pcmk_scheduler_t scheduler)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pcmk_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS, rule_data,
	parent_orig_meta, NULL, FALSE, scheduler);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, scheduler);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, scheduler);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE,
	scheduler);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *clone = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	- pe_err("No resource object for template unpacking");
	+ pcmk__config_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	- pe_err("'%s' object must have a id", xml_obj->name);
	+ pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	- pe_err("The resource object '%s' should not reference itself", id);
	+ pcmk__config_err("The resource object '%s' should not reference itself",
	+ id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//" XML_CIB_TAG_RESOURCES,
	scheduler->input, LOG_TRACE);
	if (cib_resources == NULL) {
	- pe_err("No resources configured");
	+ pcmk__config_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	XML_ATTR_ID, template_ref);
	if (template == NULL) {
	- pe_err("No template named '%s'", template_ref);
	+ pcmk__config_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, XML_ATTR_ID, id);

	clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
	if(clone) {
	crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
	}

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	#if 0 /* Disable multi-level templates for now */
	if (!unpack_template(new_xml, expanded_xml, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}
	#endif

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	- pe_err("No resource object for processing resource list of template");
	+ pcmk__config_err("No resource object for processing resource list "
	+ "of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	- pe_err("'%s' object must have a id", xml_obj->name);
	+ pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	- pe_err("The resource object '%s' should not reference itself", id);
	+ pcmk__config_err("The resource object '%s' should not reference itself",
	+ id);
	return FALSE;
	}

	if (add_tag_ref(scheduler->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pcmk_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
	/* @TODO in some future version, pe_warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p scheduler is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (scheduler == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, scheduler);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pcmk_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing
	\|pcmk_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pcmk_resource_t *rsc)
	{
	const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pe_warn_once(pcmk__wo_upstart,
	"Support for Upstart resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pe_warn_once(pcmk__wo_nagios,
	"Support for Nagios resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pcmk_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pcmk_resource_t *rsc,
	pcmk_resource_t parent, pcmk_scheduler_t scheduler)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = scheduler->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	- pe_err("Ignoring <%s> configuration without " XML_ATTR_ID,
	- xml_obj->name);
	+ pcmk__config_err("Ignoring <%s> configuration without " XML_ATTR_ID,
	+ xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pcmk_resource_t));
	if (*rsc == NULL) {
	crm_crit("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = scheduler;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, scheduler->input);

	(rsc)->variant = get_resource_type((const char ) (*rsc)->xml->name);
	if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
	- pe_err("Ignoring resource '%s' of unknown type '%s'",
	- id, (*rsc)->xml->name);
	+ pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
	+ id, (*rsc)->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, scheduler);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, scheduler); // \deprecated

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pcmk_rsc_runnable\|pcmk_rsc_unassigned);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = pcmk_role_stopped;
	(*rsc)->next_role = pcmk_role_unknown;

	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* allow-migrate=false.
	*/
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
	(*rsc)->id);
	pe_warn_once(pcmk__wo_restart_type,
	"Support for restart-type is deprecated and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_stop;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_block;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	- pe_warn("%s is not a valid value for " XML_RSC_ATTR_MULTIPLE
	- ", using default of \"stop_start\"", value);
	+ pcmk__config_warn("%s is not a valid value for "
	+ XML_RSC_ATTR_MULTIPLE
	+ ", using default of \"stop_start\"", value);
	}
	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: stop/start",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pe_warn_once(pcmk__wo_neg_threshold,
	PCMK_META_MIGRATION_THRESHOLD
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	pe__set_resource_flags(*rsc, pcmk_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	// Stored as seconds
	(*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, scheduler);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* reconnect_interval.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	(*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
	/* we want to override any default failure_timeout in use when remote
	* reconnect_interval is in use. */
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pcmk__rsc_trace(rsc, "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != pcmk_role_unknown? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default",
	scheduler);
	}

	pcmk__rsc_trace(rsc, "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, scheduler);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pcmk_resource_t child, pcmk_resource_t rsc)
	{
	pcmk_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pcmk_resource_t *
	uber_parent(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->parent != NULL)
	&& (parent->parent->variant != pcmk_rsc_variant_bundle)) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pcmk_resource_t *
	pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
	{
	const pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle
	&& (parent->parent->variant == pcmk_rsc_variant_bundle)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pcmk_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pcmk__rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {

	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pcmk__rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of pcmk_rsc_methods_t:active_node()
	static pcmk_node_t *
	active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pcmk_node_t *
	pe__find_active_requires(const pcmk_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pcmk_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pcmk_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role > pcmk_role_stopped)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pcmk_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role),
	why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
	index 0cd74dc3f4..56fa2cdfff 100644
	--- a/lib/pengine/failcounts.c
	+++ b/lib/pengine/failcounts.c
	@@ -1,470 +1,470 @@
	/*
	* Copyright 2008-2023 the Pacemaker project contributors
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/pengine/internal.h>

	static gboolean
	is_matched_failure(const char rsc_id, const xmlNode conf_op_xml,
	const xmlNode *lrm_op_xml)
	{
	gboolean matched = FALSE;
	const char *conf_op_name = NULL;
	const char *lrm_op_task = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	guint lrm_op_interval_ms = 0;
	const char *lrm_op_id = NULL;
	char *last_failure_key = NULL;

	if (rsc_id == NULL \|\| conf_op_xml == NULL \|\| lrm_op_xml == NULL) {
	return FALSE;
	}

	// Get name and interval from configured op
	conf_op_name = crm_element_value(conf_op_xml, "name");
	conf_op_interval_spec = crm_element_value(conf_op_xml,
	XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	// Get name and interval from op history entry
	lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
	crm_element_value_ms(lrm_op_xml, XML_LRM_ATTR_INTERVAL_MS,
	&lrm_op_interval_ms);

	if ((conf_op_interval_ms != lrm_op_interval_ms)
	\|\| !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
	return FALSE;
	}

	lrm_op_id = ID(lrm_op_xml);
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);

	if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
	matched = TRUE;

	} else {
	char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
	conf_op_interval_ms);

	if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
	int rc = 0;
	int target_rc = pe__target_rc_from_xml(lrm_op_xml);

	crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
	if (rc != target_rc) {
	matched = TRUE;
	}
	}
	free(expected_op_key);
	}

	free(last_failure_key);
	return matched;
	}

	static gboolean
	block_failure(const pcmk_node_t node, pcmk_resource_t rsc,
	const xmlNode *xml_op)
	{
	char *xml_name = clone_strip(rsc->id);

	/* @TODO This xpath search occurs after template expansion, but it is unable
	* to properly detect on-fail in id-ref, operation meta-attributes, or
	* op_defaults, or evaluate rules.
	*
	* Also, on-fail defaults to block (in unpack_operation()) for stop actions
	* when stonith is disabled.
	*
	* Ideally, we'd unpack the operation before this point, and pass in a
	* meta-attributes table that takes all that into consideration.
	*/
	char *xpath = crm_strdup_printf("//" XML_CIB_TAG_RESOURCE
	"[@" XML_ATTR_ID "='%s']"
	"//" XML_ATTR_OP
	"[@" XML_OP_ATTR_ON_FAIL "='block']",
	xml_name);

	xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
	gboolean should_block = FALSE;

	free(xpath);

	if (xpathObj) {
	int max = numXpathResults(xpathObj);
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *pref = getXpathResult(xpathObj, lpc);

	if (xml_op) {
	should_block = is_matched_failure(xml_name, pref, xml_op);
	if (should_block) {
	break;
	}

	} else {
	const char *conf_op_name = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	char *lrm_op_xpath = NULL;
	xmlXPathObject *lrm_op_xpathObj = NULL;

	// Get name and interval from configured op
	conf_op_name = crm_element_value(pref, "name");
	conf_op_interval_spec = crm_element_value(pref, XML_LRM_ATTR_INTERVAL);
	conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

	#define XPATH_FMT "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"//" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_TASK "='%s']" \
	"[@" XML_LRM_ATTR_INTERVAL "='%u']"

	lrm_op_xpath = crm_strdup_printf(XPATH_FMT,
	node->details->uname, xml_name,
	conf_op_name,
	conf_op_interval_ms);
	lrm_op_xpathObj = xpath_search(rsc->cluster->input, lrm_op_xpath);

	free(lrm_op_xpath);

	if (lrm_op_xpathObj) {
	int max2 = numXpathResults(lrm_op_xpathObj);
	int lpc2 = 0;

	for (lpc2 = 0; lpc2 < max2; lpc2++) {
	xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
	lpc2);

	should_block = is_matched_failure(xml_name, pref,
	lrm_op_xml);
	if (should_block) {
	break;
	}
	}
	}
	freeXpathObject(lrm_op_xpathObj);

	if (should_block) {
	break;
	}
	}
	}
	}

	free(xml_name);
	freeXpathObject(xpathObj);

	return should_block;
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? strdup(name) : clone_strip(name);
	}

	/*!
	* \internal
	* \brief Compile regular expression to match a failure-related node attribute
	*
	* \param[in] prefix Attribute prefix to match
	* \param[in] rsc_name Resource name to match as used in failure attributes
	* \param[in] is_legacy Whether DC uses per-resource fail counts
	* \param[in] is_unique Whether the resource is a globally unique clone
	* \param[out] re Where to store resulting regular expression
	*
	* \return Standard Pacemaker return code
	* \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
	* The caller is responsible for freeing re with regfree().
	*/
	static int
	generate_fail_regex(const char prefix, const char rsc_name,
	gboolean is_legacy, gboolean is_unique, regex_t *re)
	{
	char *pattern;

	/* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
	* per-operation.
	*/
	const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");

	/* Ignore instance numbers for anything other than globally unique clones.
	* Anonymous clone fail counts could contain an instance number if the
	* clone was initially unique, failed, then was converted to anonymous.
	* @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
	* clone instance numbers.
	*/
	const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");

	pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
	instance_pattern, op_pattern);
	if (regcomp(re, pattern, REG_EXTENDED\|REG_NOSUB) != 0) {
	free(pattern);
	return EINVAL;
	}

	free(pattern);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Compile regular expressions to match failure-related node attributes
	*
	* \param[in] rsc Resource being checked for failures
	* \param[out] failcount_re Storage for regular expression for fail count
	* \param[out] lastfailure_re Storage for regular expression for last failure
	*
	* \return Standard Pacemaker return code
	* \note On success, the caller is responsible for freeing the expressions with
	* regfree().
	*/
	static int
	generate_fail_regexes(const pcmk_resource_t *rsc,
	regex_t failcount_re, regex_t lastfailure_re)
	{
	int rc = pcmk_rc_ok;
	char *rsc_name = rsc_fail_name(rsc);
	const char *version = crm_element_value(rsc->cluster->input,
	XML_ATTR_CRM_VERSION);

	// @COMPAT Pacemaker <= 1.1.16 used a single fail count per resource
	gboolean is_legacy = (compare_version(version, "3.0.13") < 0);

	if (generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique),
	failcount_re) != pcmk_rc_ok) {
	rc = EINVAL;

	} else if (generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name,
	is_legacy,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique),
	lastfailure_re) != pcmk_rc_ok) {
	rc = EINVAL;
	regfree(failcount_re);
	}

	free(rsc_name);
	return rc;
	}

	// Data for fail-count-related iterators
	struct failcount_data {
	const pcmk_node_t *node;// Node to check for fail count
	pcmk_resource_t *rsc; // Resource to check for fail count
	uint32_t flags; // Fail count flags
	const xmlNode *xml_op; // History entry for expiration purposes (or NULL)
	regex_t failcount_re; // Fail count regular expression to match
	regex_t lastfailure_re; // Last failure regular expression to match
	int failcount; // Fail count so far
	time_t last_failure; // Time of most recent failure so far
	};

	/*!
	* \internal
	* \brief Update fail count and last failure appropriately for a node attribute
	*
	* \param[in] key Node attribute name
	* \param[in] value Node attribute value
	* \param[in] user_data Fail count data to update
	*/
	static void
	update_failcount_for_attr(gpointer key, gpointer value, gpointer user_data)
	{
	struct failcount_data *fc_data = user_data;

	// If this is a matching fail count attribute, update fail count
	if (regexec(&(fc_data->failcount_re), (const char *) key, 0, NULL, 0) == 0) {
	fc_data->failcount = pcmk__add_scores(fc_data->failcount,
	char2score(value));
	pcmk__rsc_trace(fc_data->rsc, "Added %s (%s) to %s fail count (now %s)",
	(const char ) key, (const char ) value,
	fc_data->rsc->id,
	pcmk_readable_score(fc_data->failcount));
	return;
	}

	// If this is a matching last failure attribute, update last failure
	if (regexec(&(fc_data->lastfailure_re), (const char *) key, 0, NULL,
	0) == 0) {
	long long last_ll;

	if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
	fc_data->last_failure = (time_t) QB_MAX(fc_data->last_failure,
	last_ll);
	}
	}
	}

	/*!
	* \internal
	* \brief Update fail count and last failure appropriately for a filler resource
	*
	* \param[in] data Filler resource
	* \param[in] user_data Fail count data to update
	*/
	static void
	update_failcount_for_filler(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *filler = data;
	struct failcount_data *fc_data = user_data;
	time_t filler_last_failure = 0;

	fc_data->failcount += pe_get_failcount(fc_data->node, filler,
	&filler_last_failure, fc_data->flags,
	fc_data->xml_op);
	fc_data->last_failure = QB_MAX(fc_data->last_failure, filler_last_failure);
	}

	/*!
	* \internal
	* \brief Get a resource's fail count on a node
	*
	* \param[in] node Node to check
	* \param[in,out] rsc Resource to check
	* \param[out] last_failure If not NULL, where to set time of most recent
	* failure of \p rsc on \p node
	* \param[in] flags Group of enum pcmk__fc_flags
	* \param[in] xml_op If not NULL, consider only the action in this
	* history entry when determining whether on-fail
	* is configured as "blocked", otherwise consider
	* all actions configured for \p rsc
	*
	* \return Fail count for \p rsc on \p node according to \p flags
	*/
	int
	pe_get_failcount(const pcmk_node_t node, pcmk_resource_t rsc,
	time_t last_failure, uint32_t flags, const xmlNode xml_op)
	{
	struct failcount_data fc_data = {
	.node = node,
	.rsc = rsc,
	.flags = flags,
	.xml_op = xml_op,
	.failcount = 0,
	.last_failure = (time_t) 0,
	};

	// Calculate resource failcount as sum of all matching operation failcounts
	CRM_CHECK(generate_fail_regexes(rsc, &fc_data.failcount_re,
	&fc_data.lastfailure_re) == pcmk_rc_ok,
	return 0);
	g_hash_table_foreach(node->details->attrs, update_failcount_for_attr,
	&fc_data);
	regfree(&(fc_data.failcount_re));
	regfree(&(fc_data.lastfailure_re));

	// If failure blocks the resource, disregard any failure timeout
	if ((fc_data.failcount > 0) && (rsc->failure_timeout > 0)
	&& block_failure(node, rsc, xml_op)) {

	- pe_warn("Ignoring failure timeout %d for %s "
	- "because it conflicts with on-fail=block",
	- rsc->failure_timeout, rsc->id);
	+ pcmk__config_warn("Ignoring failure timeout %d for %s "
	+ "because it conflicts with on-fail=block",
	+ rsc->failure_timeout, rsc->id);
	rsc->failure_timeout = 0;
	}

	// If all failures have expired, ignore fail count
	if (pcmk_is_set(flags, pcmk__fc_effective) && (fc_data.failcount > 0)
	&& (fc_data.last_failure > 0) && (rsc->failure_timeout != 0)) {

	time_t now = get_effective_time(rsc->cluster);

	if (now > (fc_data.last_failure + rsc->failure_timeout)) {
	pcmk__rsc_debug(rsc, "Failcount for %s on %s expired after %ds",
	rsc->id, pe__node_name(node), rsc->failure_timeout);
	fc_data.failcount = 0;
	}
	}

	/* Add the fail count of any filler resources, except that we never want the
	* fail counts of a bundle container's fillers to count towards the
	* container's fail count.
	*
	* Most importantly, a Pacemaker Remote connection to a bundle container
	* is a filler of the container, but can reside on a different node than the
	* container itself. Counting its fail count on its node towards the
	* container's fail count on that node could lead to attempting to stop the
	* container on the wrong node.
	*/
	if (pcmk_is_set(flags, pcmk__fc_fillers) && (rsc->fillers != NULL)
	&& !pe_rsc_is_bundled(rsc)) {

	g_list_foreach(rsc->fillers, update_failcount_for_filler, &fc_data);
	if (fc_data.failcount > 0) {
	pcmk__rsc_info(rsc,
	"Container %s and the resources within it "
	"have failed %s time%s on %s",
	rsc->id, pcmk_readable_score(fc_data.failcount),
	pcmk__plural_s(fc_data.failcount),
	pe__node_name(node));
	}

	} else if (fc_data.failcount > 0) {
	pcmk__rsc_info(rsc, "%s has failed %s time%s on %s",
	rsc->id, pcmk_readable_score(fc_data.failcount),
	pcmk__plural_s(fc_data.failcount), pe__node_name(node));
	}

	if (last_failure != NULL) {
	if ((fc_data.failcount > 0) && (fc_data.last_failure > 0)) {
	*last_failure = fc_data.last_failure;
	} else {
	*last_failure = 0;
	}
	}
	return fc_data.failcount;
	}

	/*!
	* \brief Schedule a controller operation to clear a fail count
	*
	* \param[in,out] rsc Resource with failure
	* \param[in] node Node failure occurred on
	* \param[in] reason Readable description why needed (for logging)
	* \param[in,out] scheduler Scheduler data cluster
	*
	* \return Scheduled action
	*/
	pcmk_action_t *
	pe__clear_failcount(pcmk_resource_t rsc, const pcmk_node_t node,
	const char reason, pcmk_scheduler_t scheduler)
	{
	char *key = NULL;
	pcmk_action_t *clear = NULL;

	CRM_CHECK(rsc && node && reason && scheduler, return NULL);

	key = pcmk__op_key(rsc->id, PCMK_ACTION_CLEAR_FAILCOUNT, 0);
	clear = custom_action(rsc, key, PCMK_ACTION_CLEAR_FAILCOUNT, node, FALSE,
	scheduler);
	add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
	rsc->id, pe__node_name(node), reason, clear->uuid);
	return clear;
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index c38a18739d..e1edeb70fe 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1454 +1,1454 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pcmk_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pcmk_resource_t rsc, pcmk_node_t node,
	gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == pcmk_role_promoted) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pcmk__rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pe__node_name(node), node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pcmk__rsc_trace(rsc,
	"%s now has priority %d with %s'%s' "
	"(priority: %d%s) from guest node %s",
	pe__node_name(a_node), a_node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "",
	pe__node_name(node));
	}
	}
	}

	void
	native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pcmk__rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pcmk_rsc_variant_primitive) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& node->details->maintenance) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_maintenance);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk_resource_t *p = rsc->parent;

	pcmk__rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default",
	scheduler);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	case pcmk_multiply_active_stop:
	{
	GHashTableIter gIter;
	pcmk_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(scheduler->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case pcmk_multiply_active_block:
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);

	/* If the resource belongs to a group or bundle configured with
	* multiple-active=block, block the entire entity.
	*/
	if (rsc->parent
	&& ((rsc->parent->variant == pcmk_rsc_variant_group)
	\|\| (rsc->parent->variant == pcmk_rsc_variant_bundle))
	&& (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *child = gIter->data;

	pe__clear_resource_flags(child, pcmk_rsc_managed);
	pe__set_resource_flags(child, pcmk_rsc_blocked);
	}
	}
	break;

	// pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
	default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pe__node_name(node),
	recovery2text(rsc->recovery_type));

	} else {
	pcmk__rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pe__node_name(node));
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, scheduler, FALSE);
	}
	}

	static void
	recursive_clear_unique(pcmk_resource_t *rsc, gpointer user_data)
	{
	pe__clear_resource_flags(rsc, pcmk_rsc_unique);
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *parent = uber_parent(rsc);
	const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, scheduler);

	/* Clear globally-unique on the parent and all its descendants unpacked
	* so far (clearing the parent should make any future children unpacking
	* correct). We have to clear this resource explicitly because it isn't
	* hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pcmk_rsc_promotable)) {

	- pe_err("Resource %s is of type %s and therefore "
	- "cannot be used as a promotable clone resource",
	- rsc->id, standard);
	+ pcmk__config_err("Resource %s is of type %s and therefore "
	+ "cannot be used as a promotable clone resource",
	+ rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pcmk_resource_t rsc, const pcmk_node_t node, int flags)
	{
	pcmk__rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pe__node_name(node));

	if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->running_on != NULL)) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pcmk_node_t loc = (pcmk_node_t ) iter->data;

	if (loc->details == node->details) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
	&& (rsc->running_on == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->allocated_to != NULL)
	&& (rsc->allocated_to->details == node->details)) {
	return true;
	}
	return false;
	}

	pcmk_resource_t *
	native_find_rsc(pcmk_resource_t rsc, const char id,
	const pcmk_node_t *on_node, int flags)
	{
	bool match = false;
	pcmk_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_history)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
	\|\| (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pcmk_resource_t rsc, pcmk_node_t node, gboolean create,
	const char name, pcmk_scheduler_t scheduler)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pcmk__rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, scheduler);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	pcmk__str_update(&value_copy, value);
	return value_copy;
	}

	gboolean
	native_active(pcmk_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	} else if (!a_node->details->online
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pe__node_name(a_node));
	} else {
	pcmk__rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pcmk_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_PROMOTE,
	pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_DEMOTE,
	pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(const pcmk_resource_t *rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(const pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == pcmk_role_started)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {

	role = pcmk_role_unpromoted;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pcmk_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	native_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_removed));
	status_print("blocked=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_blocked));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" " XML_ATTR_ID "=\"%s\" "
	"cached=\"%s\"/>\n",
	pre_text, pcmk__s(node->details->uname, ""),
	node->details->id, pcmk__btoa(!node->details->online));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pcmk_resource_t rsc, const char name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > pcmk_role_unpromoted) {
	pcmk__add_word(&outstr, 0, role2text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pe__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == pcmk_role_stopped) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent the
	* resource from becoming promoted).
	*/
	if (target_role_e == pcmk_role_stopped) {
	have_flags = add_output_flag(outstr, "disabled", have_flags);

	} else if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)
	&& (target_role_e == pcmk_role_unpromoted)) {
	have_flags = add_output_flag(outstr, "target-role:", have_flags);
	g_string_append(outstr, target_role);
	}
	}

	// Blocked or maintenance implies unmanaged
	if (pcmk_any_flags_set(rsc->flags,
	pcmk_rsc_blocked\|pcmk_rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t n = (pcmk_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	cl = "rsc-failed";

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t n = (pcmk_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", pe__node_name(n));

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", pe__node_name(n));

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, pe__node_name(n));

	} else {
	status_print("%s", pe__node_name(n));
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const pcmk_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	const char *desc = NULL;
	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	const char *lock_node_name = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));

	if (rsc->lock_node != NULL) {
	lock_node_name = rsc->lock_node->details->uname;
	}

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
	"id", rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pcmk_rsc_removed),
	"blocked", pe__rsc_bool_str(rsc, pcmk_rsc_blocked),
	"maintenance", pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL),
	"locked_to", lock_node_name,
	"description", desc);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	"name", node->details->uname,
	"id", node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pcmk_resource_t * rsc)
	{
	pcmk__rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pcmk__rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pcmk_node_t *
	native_location(const pcmk_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pcmk__rsc_node argument instead of int current
	pcmk_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	const pcmk_resource_t *up = pe__const_top_resource(rsc, true);

	return up->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
	return 1U;
	}
	diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
	index e3871b6b94..ce74b4c9e8 100644
	--- a/lib/pengine/pe_actions.c
	+++ b/lib/pengine/pe_actions.c
	@@ -1,1875 +1,1875 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml_internal.h>
	#include "pe_status_private.h"

	static void unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms);

	static void
	add_singleton(pcmk_scheduler_t scheduler, pcmk_action_t action)
	{
	if (scheduler->singletons == NULL) {
	scheduler->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(scheduler->singletons, action->uuid, action);
	}

	static pcmk_action_t *
	lookup_singleton(pcmk_scheduler_t scheduler, const char action_uuid)
	{
	if (scheduler->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(scheduler->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] scheduler Scheduler data
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pcmk_action_t *
	find_existing_action(const char key, const pcmk_resource_t rsc,
	const pcmk_node_t node, const pcmk_scheduler_t scheduler)
	{
	GList *matches = NULL;
	pcmk_action_t *action = NULL;

	/* When rsc is NULL, it would be quicker to check scheduler->singletons,
	* but checking all scheduler->actions takes the node into account.
	*/
	matches = find_actions(((rsc == NULL)? scheduler->actions : rsc->actions),
	key, node);
	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	/*!
	* \internal
	* \brief Find the XML configuration corresponding to a specific action key
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] key "RSC_ACTION_INTERVAL" of action to find
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	static xmlNode *
	find_exact_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {

	bool enabled = false;
	const char *config_name = NULL;
	const char *interval_spec = NULL;

	// @TODO This does not consider rules, defaults, etc.
	if (!include_disabled
	&& (pcmk__xe_get_bool_attr(operation, "enabled",
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (crm_parse_interval_spec(interval_spec) != interval_ms) {
	continue;
	}

	config_name = crm_element_value(operation, "name");
	if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
	return operation;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Find the XML configuration of a resource action
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] action_name Action name to search for
	* \param[in] interval_ms Action interval (in milliseconds) to search for
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	xmlNode *
	pcmk__find_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	xmlNode *action_config = NULL;

	// Try requested action first
	action_config = find_exact_action_config(rsc, action_name, interval_ms,
	include_disabled);

	// For migrate_to and migrate_from actions, retry with "migrate"
	// @TODO This should be either documented or deprecated
	if ((action_config == NULL)
	&& pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	action_config = find_exact_action_config(rsc, "migrate", 0,
	include_disabled);
	}

	return action_config;
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pcmk_action_t *
	new_action(char key, const char task, pcmk_resource_t *rsc,
	const pcmk_node_t node, bool optional, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *action = calloc(1, sizeof(pcmk_action_t));

	CRM_ASSERT(action != NULL);

	action->rsc = rsc;
	action->task = strdup(task); CRM_ASSERT(action->task != NULL);
	action->uuid = key;

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pe__set_action_flags(action, pcmk_action_on_dc);
	}

	pe__set_action_flags(action, pcmk_action_runnable);
	if (optional) {
	pe__set_action_flags(action, pcmk_action_optional);
	} else {
	pe__clear_action_flags(action, pcmk_action_optional);
	}

	if (rsc == NULL) {
	action->meta = pcmk__strkey_table(free, free);
	} else {
	guint interval_ms = 0;

	parse_op_key(key, NULL, NULL, &interval_ms);
	action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
	true);

	/* If the given key is for one of the many notification pseudo-actions
	* (pre_notify_promote, etc.), the actual action name is "notify"
	*/
	if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
	action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
	0, true);
	}

	unpack_operation(action, action->op_entry, interval_ms);
	}

	pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	scheduler->action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	pe__node_name(node));
	action->id = scheduler->action_id++;

	scheduler->actions = g_list_prepend(scheduler->actions, action);
	if (rsc == NULL) {
	add_singleton(scheduler, action);
	} else {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Unpack a resource's action-specific instance parameters
	*
	* \param[in] action_xml XML of action's configuration in CIB (if any)
	* \param[in,out] node_attrs Table of node attributes (for rule evaluation)
	* \param[in,out] scheduler Cluster working set (for rule evaluation)
	*
	* \return Newly allocated hash table of action-specific instance parameters
	*/
	GHashTable *
	pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = node_attrs,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__unpack_dataset_nvpairs(action_xml, XML_TAG_ATTR_SETS,
	&rule_data, params, NULL,
	FALSE, scheduler);
	return params;
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in,out] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pcmk_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& (g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is optional (%s is unmanaged)",
	action->uuid, pe__node_name(action->node),
	action->rsc->id);
	pe__set_action_flags(action, pcmk_action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pe__clear_action_flags(action, pcmk_action_optional);
	}
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	enum pe_quorum_policy policy = scheduler->no_quorum_policy;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	policy = pcmk_no_quorum_ignore;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
	switch (rsc->role) {
	case pcmk_role_promoted:
	case pcmk_role_unpromoted:
	if (rsc->next_role > pcmk_role_unpromoted) {
	pe__set_next_role(rsc, pcmk_role_unpromoted,
	"no-quorum-policy=demote");
	}
	policy = pcmk_no_quorum_ignore;
	break;
	default:
	policy = pcmk_no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in,out] action Action to update
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pcmk_action_t *action,
	pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pcmk__rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
	action->uuid);
	pe__clear_action_flags(action, pcmk_action_runnable);

	} else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
	&& !(action->node->details->online)
	&& (!pe__is_guest_node(action->node)
	\|\| action->node->details->remote_requires_reset)) {
	pe__clear_action_flags(action, pcmk_action_runnable);
	do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
	action->uuid, pe__node_name(action->node));
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
	&& action->node->details->pending) {
	pe__clear_action_flags(action, pcmk_action_runnable);
	do_crm_log(LOG_WARNING,
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, pe__node_name(action->node));

	} else if (action->needs == pcmk_requires_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pe__is_guest_node(action->node)
	&& !pe_can_fence(scheduler, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pcmk_action_runnable);
	} else {
	pcmk__rsc_trace(action->rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, pe__node_name(action->node));
	pe__set_action_flags(action, pcmk_action_runnable);
	}

	} else {
	switch (effective_quorum_policy(action->rsc, scheduler)) {
	case pcmk_no_quorum_stop:
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case pcmk_no_quorum_freeze:
	if (!action->rsc->fns->active(action->rsc, TRUE)
	\|\| (action->rsc->next_role > action->rsc->role)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pe__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pe__set_action_flags(action, pcmk_action_runnable);
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource object's flags for a new action on it
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] action New action
	*/
	static void
	update_resource_flags_for_action(pcmk_resource_t *rsc,
	const pcmk_action_t *action)
	{
	/* @COMPAT pcmk_rsc_starting and pcmk_rsc_stopping are deprecated and unused
	* within Pacemaker, and will eventually be removed
	*/
	if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_stopping);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) {
	if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pe__set_resource_flags(rsc, pcmk_rsc_starting);
	} else {
	pe__clear_resource_flags(rsc, pcmk_rsc_starting);
	}
	}
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
	}

	/*!
	* \internal
	* \brief Validate (and possibly reset) resource action's on_fail meta-attribute
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] action_config Action configuration XML from CIB (if any)
	* \param[in,out] meta Table of action meta-attributes
	*/
	static void
	validate_on_fail(const pcmk_resource_t rsc, const char action_name,
	const xmlNode action_config, GHashTable meta)
	{
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
	char *key = NULL;
	char *new_value = NULL;

	// Stop actions can only use certain on-fail values
	if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", rsc->id, value);
	g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
	return;
	}

	/* Demote actions default on-fail to the on-fail value for the first
	* recurring monitor for the promoted role (if any).
	*/
	if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
	&& (value == NULL)) {

	/* @TODO This does not consider promote options set in a meta-attribute
	* block (which may have rules that need to be evaluated) rather than
	* XML properties.
	*/
	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {
	bool enabled = false;
	const char *promote_on_fail = NULL;

	/* We only care about explicit on-fail (if promote uses default, so
	* can demote)
	*/
	promote_on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	if (promote_on_fail == NULL) {
	continue;
	}

	// We only care about recurring monitors for the promoted role
	name = crm_element_value(operation, "name");
	role = crm_element_value(operation, "role");
	if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
	continue;
	}
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (crm_parse_interval_spec(interval_spec) == 0) {
	continue;
	}

	// We only care about enabled monitors
	if ((pcmk__xe_get_bool_attr(operation, "enabled",
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	// Demote actions can't default to on-fail="demote"
	if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) {
	continue;
	}

	// Use value from first applicable promote action found
	key = strdup(XML_OP_ATTR_ON_FAIL);
	new_value = strdup(promote_on_fail);
	CRM_ASSERT((key != NULL) && (new_value != NULL));
	g_hash_table_insert(meta, key, new_value);
	}
	return;
	}

	if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
	&& !pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	key = strdup(XML_OP_ATTR_ON_FAIL);
	new_value = strdup("ignore");
	CRM_ASSERT((key != NULL) && (new_value != NULL));
	g_hash_table_insert(meta, key, new_value);
	return;
	}

	// on-fail="demote" is allowed only for certain actions
	if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	name = crm_element_value(action_config, "name");
	role = crm_element_value(action_config, "role");
	interval_spec = crm_element_value(action_config,
	XML_LRM_ATTR_INTERVAL);

	if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
	&& (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)
	\|\| (crm_parse_interval_spec(interval_spec) == 0))) {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", rsc->id, name);
	g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
	return;
	}
	}
	}

	static int
	unpack_timeout(const char *value)
	{
	int timeout_ms = crm_get_msec(value);

	if (timeout_ms < 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}
	return timeout_ms;
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, const xmlNode xml_obj,
	guint interval_ms, const crm_time_t *now,
	long long *start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
	"'%s' because '%s' is not valid",
	(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result,
	(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	int start_delay = 0;

	if (value != NULL) {
	start_delay = crm_get_msec(value);

	if (start_delay < 0) {
	start_delay = 0;
	}

	if (meta) {
	g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
	pcmk__itoa(start_delay));
	}
	}

	return start_delay;
	}

	/*!
	* \internal
	* \brief Find a resource's most frequent recurring monitor
	*
	* \param[in] rsc Resource to check
	*
	* \return Operation XML configured for most frequent recurring monitor for
	* \p rsc (if any)
	*/
	static xmlNode *
	most_frequent_monitor(const pcmk_resource_t *rsc)
	{
	guint min_interval_ms = G_MAXUINT;
	xmlNode *op = NULL;

	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {
	bool enabled = false;
	guint interval_ms = 0;
	const char *interval_spec = crm_element_value(operation,
	XML_LRM_ATTR_INTERVAL);

	// We only care about enabled recurring monitors
	if (!pcmk__str_eq(crm_element_value(operation, "name"),
	PCMK_ACTION_MONITOR, pcmk__str_none)) {
	continue;
	}
	interval_ms = crm_parse_interval_spec(interval_spec);
	if (interval_ms == 0) {
	continue;
	}

	// @TODO This does not account for rules, defaults, etc.
	if ((pcmk__xe_get_bool_attr(operation, "enabled",
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	if (interval_ms < min_interval_ms) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	return op;
	}

	/*!
	* \internal
	* \brief Unpack action meta-attributes
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node that action is on
	* \param[in] action_name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] action_config Action XML configuration from CIB (if any)
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds) from its CIB XML
	* configuration (including defaults).
	*
	* \return Newly allocated hash table with normalized action meta-attributes
	*/
	GHashTable *
	pcmk__unpack_action_meta(pcmk_resource_t rsc, const pcmk_node_t node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config)
	{
	GHashTable *meta = NULL;
	char *name = NULL;
	char *value = NULL;
	const char *timeout_spec = NULL;
	const char *str = NULL;

	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE),
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action_name,
	.interval = interval_ms,
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = (node == NULL)? NULL : node->details->attrs,
	.role = pcmk_role_unknown,
	.now = rsc->cluster->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data,
	};

	meta = pcmk__strkey_table(free, free);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(rsc->cluster->op_defaults, XML_TAG_META_SETS,
	&rule_data, meta, NULL, FALSE, rsc->cluster);

	// Derive default timeout for probes from recurring monitor timeouts
	if (pcmk_is_probe(action_name, interval_ms)) {
	xmlNode *min_interval_mon = most_frequent_monitor(rsc);

	if (min_interval_mon != NULL) {
	/* @TODO This does not consider timeouts set in meta_attributes
	* blocks (which may also have rules that need to be evaluated).
	*/
	timeout_spec = crm_element_value(min_interval_mon,
	XML_ATTR_TIMEOUT);
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting default timeout for %s probe to "
	"most frequent monitor's timeout '%s'",
	rsc->id, timeout_spec);
	name = strdup(XML_ATTR_TIMEOUT);
	value = strdup(timeout_spec);
	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}
	}

	if (action_config != NULL) {
	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(action_config, XML_TAG_META_SETS, &rule_data,
	meta, NULL, TRUE, rsc->cluster);

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	* (See below for the only exception, fence device start/probe timeout.)
	*/
	for (xmlAttrPtr attr = action_config->properties;
	attr != NULL; attr = attr->next) {
	name = strdup((const char *) attr->name);
	value = strdup(pcmk__xml_attr_value(attr));

	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}

	g_hash_table_remove(meta, XML_ATTR_ID);

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	name = strdup(XML_LRM_ATTR_INTERVAL);
	CRM_ASSERT(name != NULL);
	value = crm_strdup_printf("%u", interval_ms);
	g_hash_table_insert(meta, name, value);
	} else {
	g_hash_table_remove(meta, XML_LRM_ATTR_INTERVAL);
	}

	/* Timeout order of precedence (highest to lowest):
	* 1. pcmk_monitor_timeout resource parameter (only for starts and probes
	* when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
	* monitors via the executor instead)
	* 2. timeout configured in <op> (with <op timeout> taking precedence over
	* <op> <meta_attributes>)
	* 3. timeout configured in <op_defaults> <meta_attributes>
	* 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
	*/

	// Check for pcmk_monitor_timeout
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
	\|\| pcmk_is_probe(action_name, interval_ms))) {

	GHashTable *params = pe_rsc_params(rsc, node, rsc->cluster);

	timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting timeout for %s %s to "
	"pcmk_monitor_timeout (%s)",
	rsc->id, action_name, timeout_spec);
	name = strdup(XML_ATTR_TIMEOUT);
	value = strdup(timeout_spec);
	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}

	// Normalize timeout to positive milliseconds
	name = strdup(XML_ATTR_TIMEOUT);
	CRM_ASSERT(name != NULL);
	timeout_spec = g_hash_table_lookup(meta, XML_ATTR_TIMEOUT);
	g_hash_table_insert(meta, name, pcmk__itoa(unpack_timeout(timeout_spec)));

	// Ensure on-fail has a valid value
	validate_on_fail(rsc, action_name, action_config, meta);

	// Normalize start-delay
	str = g_hash_table_lookup(meta, XML_OP_ATTR_START_DELAY);
	if (str != NULL) {
	unpack_start_delay(str, meta);
	} else {
	long long start_delay = 0;

	str = g_hash_table_lookup(meta, XML_OP_ATTR_ORIGIN);
	if (unpack_interval_origin(str, action_config, interval_ms,
	rsc->cluster->now, &start_delay)) {
	name = strdup(XML_OP_ATTR_START_DELAY);
	CRM_ASSERT(name != NULL);
	g_hash_table_insert(meta, name,
	crm_strdup_printf("%lld", start_delay));
	}
	}
	return meta;
	}

	/*!
	* \internal
	* \brief Determine an action's quorum and fencing dependency
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action being unpacked
	*
	* \return Quorum and fencing dependency appropriate to action
	*/
	enum rsc_start_requirement
	pcmk__action_requires(const pcmk_resource_t rsc, const char action_name)
	{
	const char *value = NULL;
	enum rsc_start_requirement requires = pcmk_requires_nothing;

	CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);

	if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	requires = pcmk_requires_fencing;
	value = "fencing";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_quorum)) {
	requires = pcmk_requires_quorum;
	value = "quorum";

	} else {
	value = "nothing";
	}
	pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
	return requires;
	}

	/*!
	* \internal
	* \brief Parse action failure response from a user-provided string
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] value User-provided configuration value for on-fail
	*
	* \return Action failure response parsed from \p text
	*/
	enum action_fail_response
	pcmk__parse_on_fail(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, const char *value)
	{
	const char *desc = NULL;
	bool needs_remote_reset = false;
	enum action_fail_response on_fail = pcmk_on_fail_ignore;

	if (value == NULL) {
	// Use default

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_block;
	desc = "block";

	} else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	on_fail = pcmk_on_fail_fence_node;
	desc = "node fencing";
	} else {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
	"%s of %s to 'stop' because 'fence' is not "
	"valid when fencing is disabled",
	action_name, rsc->id);
	on_fail = pcmk_on_fail_stop;
	desc = "stop resource";
	}

	} else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_standby_node;
	desc = "node standby";

	} else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING,
	NULL)) {
	desc = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_ban;
	desc = "force migration";

	} else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_stop;
	desc = "stop resource";

	} else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_restart;
	desc = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
	if (rsc->container == NULL) {
	pcmk__rsc_debug(rsc,
	"Using default " XML_OP_ATTR_ON_FAIL " for %s "
	"of %s because it does not have a container",
	action_name, rsc->id);
	} else {
	on_fail = pcmk_on_fail_restart_container;
	desc = "restart container (and possibly migrate)";
	}

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_demote;
	desc = "demote instance";

	} else {
	pcmk__config_err("Using default '" XML_OP_ATTR_ON_FAIL "' for "
	"%s of %s because '%s' is not valid",
	action_name, rsc->id, value);
	}

	/* Remote node connections are handled specially. Failures that result
	* in dropping an active connection must result in fencing. The only
	* failures that don't are probes and starts. The user can explicitly set
	* on-fail="fence" to fence after start failures.
	*/
	if (pe__resource_is_remote_conn(rsc)
	&& !pcmk_is_probe(action_name, interval_ms)
	&& !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
	needs_remote_reset = true;
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	desc = NULL; // Force default for unmanaged connections
	}
	}

	if (desc != NULL) {
	// Explicit value used, default not needed

	} else if (rsc->container != NULL) {
	on_fail = pcmk_on_fail_restart_container;
	desc = "restart container (and possibly migrate) (default)";

	} else if (needs_remote_reset) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	desc = "fence remote node (default)";
	} else {
	desc = "recover remote node connection (default)";
	}
	on_fail = pcmk_on_fail_reset_remote;
	} else {
	on_fail = pcmk_on_fail_stop;
	desc = "stop unmanaged remote node (enforcing default)";
	}

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	on_fail = pcmk_on_fail_fence_node;
	desc = "resource fence (default)";
	} else {
	on_fail = pcmk_on_fail_block;
	desc = "resource block (default)";
	}

	} else {
	on_fail = pcmk_on_fail_restart;
	desc = "restart (and possibly migrate) (default)";
	}

	pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
	pcmk__readable_interval(interval_ms), action_name,
	rsc->id, desc);
	return on_fail;
	}

	/*!
	* \internal
	* \brief Determine a resource's role after failure of an action
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] on_fail Failure handling for action
	* \param[in] meta Unpacked action meta-attributes
	*
	* \return Resource role that results from failure of action
	*/
	enum rsc_role_e
	pcmk__role_after_failure(const pcmk_resource_t rsc, const char action_name,
	enum action_fail_response on_fail, GHashTable *meta)
	{
	const char *value = NULL;
	enum rsc_role_e role = pcmk_role_unknown;

	// Set default for role after failure specially in certain circumstances
	switch (on_fail) {
	case pcmk_on_fail_stop:
	role = pcmk_role_stopped;
	break;

	case pcmk_on_fail_reset_remote:
	if (rsc->remote_reconnect_ms != 0) {
	role = pcmk_role_stopped;
	}
	break;

	default:
	break;
	}

	// @COMPAT Check for explicitly configured role (deprecated)
	value = g_hash_table_lookup(meta, "role_after_failure");
	if (value != NULL) {
	pe_warn_once(pcmk__wo_role_after,
	"Support for role_after_failure is deprecated "
	"and will be removed in a future release");
	if (role == pcmk_role_unknown) {
	role = text2role(value);
	}
	}

	if (role == pcmk_role_unknown) {
	// Use default
	if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	role = pcmk_role_unpromoted;
	} else {
	role = pcmk_role_started;
	}
	}
	pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
	rsc->id, action_name, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief Unpack action configuration
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds), requirements, and
	* failure policy from its CIB XML configuration (including defaults).
	*
	* \param[in,out] action Resource action to unpack into
	* \param[in] xml_obj Action configuration XML (NULL for defaults only)
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms)
	{
	const char *value = NULL;

	action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
	action->task, interval_ms, xml_obj);
	action->needs = pcmk__action_requires(action->rsc, action->task);

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
	action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
	interval_ms, value);

	action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
	action->on_fail, action->meta);
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in,out] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Action object corresponding to arguments (guaranteed not to be
	* \c NULL)
	* \note This function takes ownership of (and might free) \p key, and
	* \p scheduler takes ownership of the returned action (the caller should
	* not free it).
	*/
	pcmk_action_t *
	custom_action(pcmk_resource_t rsc, char key, const char *task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL));

	action = find_existing_action(key, rsc, on_node, scheduler);
	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, scheduler);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	if ((action->node != NULL) && (action->op_entry != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_attrs_evaluated)) {

	GHashTable *attrs = action->node->details->attrs;

	if (action->extra != NULL) {
	g_hash_table_destroy(action->extra);
	}
	action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
	attrs, scheduler);
	pe__set_action_flags(action, pcmk_action_attrs_evaluated);
	}

	update_resource_action_runnable(action, scheduler);
	update_resource_flags_for_action(rsc, action);
	}

	if (action->extra == NULL) {
	action->extra = pcmk__strkey_table(free, free);
	}

	return action;
	}

	pcmk_action_t *
	get_pseudo_op(const char name, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *op = lookup_singleton(scheduler, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
	pe__set_action_flags(op, pcmk_action_pseudo\|pcmk_action_runnable);
	}
	return op;
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *candidate = gIter->data;

	if (candidate->children != NULL) {
	matches = find_unfencing_devices(candidate->children, matches);

	} else if (!pcmk_is_set(candidate->flags, pcmk_rsc_fence_device)) {
	continue;

	} else if (pcmk_is_set(candidate->flags, pcmk_rsc_needs_unfencing)) {
	matches = g_list_prepend(matches, candidate);

	} else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta,
	PCMK_STONITH_PROVIDES),
	PCMK__VALUE_UNFENCING,
	pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(const pcmk_node_t *node,
	const pcmk_scheduler_t *scheduler)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// `priority-fencing-delay` is disabled
	if (scheduler->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->details->type != pcmk_node_variant_cluster) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *n = gIter->data;

	if (n->details->type != pcmk_node_variant_cluster) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->details->priority > top_priority) {
	top_priority = n->details->priority;
	}

	if (member_count == 1
	\|\| n->details->priority < lowest_priority) {
	lowest_priority = n->details->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->details->priority < top_priority) {
	return 0;
	}

	return scheduler->priority_fencing_delay;
	}

	pcmk_action_t *
	pe_fence_op(pcmk_node_t node, const char op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler)
	{
	char *op_key = NULL;
	pcmk_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = scheduler->stonith_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s",
	PCMK_ACTION_STONITH, node->details->uname, op);

	stonith_op = lookup_singleton(scheduler, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
	TRUE, scheduler);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
	/* Extra work to detect device changes
	*/
	GString *digests_all = g_string_sized_new(1024);
	GString *digests_secure = g_string_sized_new(1024);

	GList *matches = find_unfencing_devices(scheduler->resources, NULL);

	char *key = NULL;
	char *value = NULL;

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->meta,
	XML_ATTR_TYPE);
	pcmk__op_digest_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node,
	scheduler);
	if (data->rc == pcmk__digest_mismatch) {
	optional = FALSE;
	crm_notice("Unfencing node %s because the definition of "
	"%s changed", pe__node_name(node), match->id);
	if (!pcmk__is_daemon && scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;

	out->info(out,
	"notice: Unfencing node %s because the "
	"definition of %s changed",
	pe__node_name(node), match->id);
	}
	}

	pcmk__g_strcat(digests_all,
	match->id, ":", agent, ":",
	data->digest_all_calc, ",", NULL);
	pcmk__g_strcat(digests_secure,
	match->id, ":", agent, ":",
	data->digest_secure_calc, ",", NULL);
	}
	key = strdup(XML_OP_ATTR_DIGESTS_ALL);
	value = strdup((const char *) digests_all->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_all, TRUE);

	key = strdup(XML_OP_ATTR_DIGESTS_SECURE);
	value = strdup((const char *) digests_secure->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_secure, TRUE);
	}

	} else {
	free(op_key);
	}

	if (scheduler->priority_fencing_delay > 0

	/* It's a suitable case where `priority-fencing-delay` applies.
	* At least add `priority-fencing-delay` field as an indicator. */
	&& (priority_delay

	/* The priority delay needs to be recalculated if this function has
	* been called by schedule_fencing_and_shutdowns() after node
	* priority has already been calculated by native_add_running().
	*/
	\|\| g_hash_table_lookup(stonith_op->meta,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add `priority-fencing-delay` to the fencing op even if it's 0 for
	* the targeting node. So that it takes precedence over any possible
	* `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
	scheduler));

	g_hash_table_insert(stonith_op->meta,
	strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(scheduler, node)) {
	pe__clear_action_flags(stonith_op, pcmk_action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	pe_free_action(pcmk_action_t *action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free);
	g_list_free_full(action->actions_after, free);
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	int
	pe_get_configured_timeout(pcmk_resource_t rsc, const char action,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *child = NULL;
	GHashTable *action_meta = NULL;
	const char *timeout_spec = NULL;
	int timeout_ms = 0;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	child != NULL; child = crm_next_same_xml(child)) {
	if (pcmk__str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME),
	pcmk__str_casei)) {
	timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
	break;
	}
	}

	if (timeout_spec == NULL && scheduler->op_defaults) {
	action_meta = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(scheduler->op_defaults, XML_TAG_META_SETS,
	&rule_data, action_meta, NULL, FALSE,
	scheduler);
	timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
	}

	// @TODO check meta-attributes
	// @TODO maybe use min-interval monitor timeout as default for monitors

	timeout_ms = crm_get_msec(timeout_spec);
	if (timeout_ms < 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	if (action_meta != NULL) {
	g_hash_table_destroy(action_meta);
	}
	return timeout_ms;
	}

	enum action_tasks
	get_complex_task(const pcmk_resource_t rsc, const char name)
	{
	enum action_tasks task = text2task(name);

	if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) {
	switch (task) {
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_demoted:
	case pcmk_action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s",
	name, rsc->id);
	--task;
	break;
	default:
	break;
	}
	}
	return task;
	}

	/*!
	* \internal
	* \brief Find first matching action in a list
	*
	* \param[in] input List of actions to search
	* \param[in] uuid If not NULL, action must have this UUID
	* \param[in] task If not NULL, action must have this action name
	* \param[in] on_node If not NULL, action must be on this node
	*
	* \return First action in list that matches criteria, or NULL if none
	*/
	pcmk_action_t *
	find_first_action(const GList input, const char uuid, const char *task,
	const pcmk_node_t *on_node)
	{
	CRM_CHECK(uuid \|\| task, return NULL);

	for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, pe__node_name(on_node));

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__str_eq(on_node->details->id, action->node->details->id,
	pcmk__str_casei)) {

	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->actions, key, node);
	} else {
	result = find_actions(rsc->actions, key, node);
	}
	free(key);
	return result;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pcmk_action_runnable:
	change = "unrunnable";
	break;
	case pcmk_action_migratable:
	change = "unmigrateable";
	break;
	case pcmk_action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason,
	pcmk__s(reason, "(none)"));
	} else if (action->reason == NULL) {
	pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, pcmk__s(reason, "(none)"));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in,out] rsc Resource to clear
	* \param[in] node Node to clear history on
	*/
	void
	pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	CRM_ASSERT((rsc != NULL) && (node != NULL));

	custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->cluster);
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	int
	pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
	const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);

	const char *a_node = crm_element_value(xml_a, XML_LRM_ATTR_TARGET);
	const char *b_node = crm_element_value(xml_b, XML_LRM_ATTR_TARGET);
	bool same_node = true;

	/* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via
	* 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c).
	*
	* In case that any of the lrm_rsc_op entries doesn't have on_node
	* attribute, we need to explicitly tell whether the two operations are on
	* the same node.
	*/
	if (a_node == NULL \|\| b_node == NULL) {
	same_node = same_node_default;

	} else {
	same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
	}

	if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	- pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	+ pcmk__config_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (a_call_id >= 0 && b_call_id >= 0
	&& (!same_node \|\| a_call_id == b_call_id)) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));
	}

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	return pe__is_newer_op(xml_a, xml_b, true);
	}

	/*!
	* \internal
	* \brief Create a new pseudo-action for a resource
	*
	* \param[in,out] rsc Resource to create action for
	* \param[in] task Action name
	* \param[in] optional Whether action should be considered optional
	* \param[in] runnable Whethe action should be considered runnable
	*
	* \return New action object corresponding to arguments
	*/
	pcmk_action_t *
	pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char task, bool optional,
	bool runnable)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL));

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
	optional, rsc->cluster);
	pe__set_action_flags(action, pcmk_action_pseudo);
	if (runnable) {
	pe__set_action_flags(action, pcmk_action_runnable);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Add the expected result to an action
	*
	* \param[in,out] action Action to add expected result to
	* \param[in] expected_result Expected result to add
	*
	* \note This is more efficient than calling add_hash_param().
	*/
	void
	pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
	{
	char *name = NULL;

	CRM_ASSERT((action != NULL) && (action->meta != NULL));

	name = strdup(XML_ATTR_TE_TARGET_RC);
	CRM_ASSERT (name != NULL);

	g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result));
	}
	diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c
	index 60b7247f0d..11ed673cfb 100644
	--- a/lib/pengine/rules.c
	+++ b/lib/pengine/rules.c
	@@ -1,1319 +1,1319 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/pengine/internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <ctype.h>

	CRM_TRACE_INIT_DATA(pe_rules);

	/*!
	* \brief Evaluate any rules contained by given XML element
	*
	* \param[in,out] xml XML element to check for rules
	* \param[in] node_hash Node attributes to use to evaluate expressions
	* \param[in] now Time to use when evaluating expressions
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if no rules, or any of rules present is in effect, else FALSE
	*/
	gboolean
	pe_evaluate_rules(xmlNode ruleset, GHashTable node_hash, crm_time_t *now,
	crm_time_t *next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_rules(ruleset, &rule_data, next_change);
	}

	gboolean
	pe_test_rule(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_expr(rule, &rule_data, next_change);
	}

	/*!
	* \brief Evaluate one rule subelement (pass/fail)
	*
	* A rule element may contain another rule, a node attribute expression, or a
	* date expression. Given any one of those, evaluate it and return whether it
	* passed.
	*
	* \param[in,out] expr Rule subelement XML
	* \param[in] node_hash Node attributes to use when evaluating expression
	* \param[in] role Resource role to use when evaluating expression
	* \param[in] now Time to use when evaluating expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	* \param[in] match_data If not NULL, resource back-references and params
	*
	* \return TRUE if expression is in effect under given conditions, else FALSE
	*/
	gboolean
	pe_test_expression(xmlNode expr, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_subexpr(expr, &rule_data, next_change);
	}

	enum expression_type
	find_expression_type(xmlNode * expr)
	{
	const char *attr = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);

	if (pcmk__xe_is(expr, PCMK_XE_DATE_EXPRESSION)) {
	return pcmk__subexpr_datetime;

	} else if (pcmk__xe_is(expr, PCMK_XE_RSC_EXPRESSION)) {
	return pcmk__subexpr_resource;

	} else if (pcmk__xe_is(expr, PCMK_XE_OP_EXPRESSION)) {
	return pcmk__subexpr_operation;

	} else if (pcmk__xe_is(expr, XML_TAG_RULE)) {
	return pcmk__subexpr_rule;

	} else if (!pcmk__xe_is(expr, XML_TAG_EXPRESSION)) {
	return pcmk__subexpr_unknown;

	} else if (pcmk__str_any_of(attr, CRM_ATTR_UNAME, CRM_ATTR_KIND, CRM_ATTR_ID, NULL)) {
	return pcmk__subexpr_location;

	} else if (pcmk__str_eq(attr, CRM_ATTR_ROLE, pcmk__str_none)) {
	return pcmk__subexpr_role;
	}

	return pcmk__subexpr_attribute;
	}

	/* As per the nethack rules:
	*
	* moon period = 29.53058 days ~= 30, year = 365.2422 days
	* days moon phase advances on first day of year compared to preceding year
	* = 365.2422 - 12*29.53058 ~= 11
	* years in Metonic cycle (time until same phases fall on the same days of
	* the month) = 18.6 ~= 19
	* moon phase on first day of year (epact) ~= (11*(year%19) + 29) % 30
	* (29 as initial condition)
	* current phase in days = first day phase + days elapsed in year
	* 6 moons ~= 177 days
	* 177 ~= 8 reported phases * 22
	* + 11/22 for rounding
	*
	* 0-7, with 0: new, 4: full
	*/

	static int
	phase_of_the_moon(const crm_time_t *now)
	{
	uint32_t epact, diy, goldn;
	uint32_t y;

	crm_time_get_ordinal(now, &y, &diy);

	goldn = (y % 19) + 1;
	epact = (11 * goldn + 18) % 30;
	if ((epact == 25 && goldn > 11) \|\| epact == 24)
	epact++;

	return ((((((diy + epact) * 6) + 11) % 177) / 22) & 7);
	}

	static int
	check_one(const xmlNode cron_spec, const char xml_field, uint32_t time_field)
	{
	int rc = pcmk_rc_undetermined;
	const char *value = crm_element_value(cron_spec, xml_field);
	long long low, high;

	if (value == NULL) {
	/* Return pe_date_result_undetermined if the field is missing. */
	goto bail;
	}

	if (pcmk__parse_ll_range(value, &low, &high) != pcmk_rc_ok) {
	goto bail;
	} else if (low == high) {
	/* A single number was given, not a range. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low != -1 && high != -1) {
	/* This is a range with both bounds. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low == -1) {
	/* This is a range with no starting value. */
	rc = time_field <= high ? pcmk_rc_within_range : pcmk_rc_after_range;
	} else if (high == -1) {
	/* This is a range with no ending value. */
	rc = time_field >= low ? pcmk_rc_within_range : pcmk_rc_before_range;
	}

	bail:
	if (rc == pcmk_rc_within_range) {
	crm_debug("Condition '%s' in %s: passed", value, xml_field);
	} else {
	crm_debug("Condition '%s' in %s: failed", value, xml_field);
	}

	return rc;
	}

	static gboolean
	check_passes(int rc) {
	/* _within_range is obvious. _undetermined is a pass because
	* this is the return value if a field is not given. In this
	* case, we just want to ignore it and check other fields to
	* see if they place some restriction on what can pass.
	*/
	return rc == pcmk_rc_within_range \|\| rc == pcmk_rc_undetermined;
	}

	#define CHECK_ONE(spec, name, var) do { \
	int subpart_rc = check_one(spec, name, var); \
	if (check_passes(subpart_rc) == FALSE) { \
	return subpart_rc; \
	} \
	} while (0)

	int
	pe_cron_range_satisfied(const crm_time_t now, const xmlNode cron_spec)
	{
	uint32_t h, m, s, y, d, w;

	CRM_CHECK(now != NULL, return pcmk_rc_op_unsatisfied);

	crm_time_get_gregorian(now, &y, &m, &d);
	CHECK_ONE(cron_spec, "years", y);
	CHECK_ONE(cron_spec, "months", m);
	CHECK_ONE(cron_spec, "monthdays", d);

	crm_time_get_timeofday(now, &h, &m, &s);
	CHECK_ONE(cron_spec, "hours", h);
	CHECK_ONE(cron_spec, "minutes", m);
	CHECK_ONE(cron_spec, "seconds", s);

	crm_time_get_ordinal(now, &y, &d);
	CHECK_ONE(cron_spec, "yeardays", d);

	crm_time_get_isoweek(now, &y, &w, &d);
	CHECK_ONE(cron_spec, "weekyears", y);
	CHECK_ONE(cron_spec, "weeks", w);
	CHECK_ONE(cron_spec, "weekdays", d);

	CHECK_ONE(cron_spec, "moon", phase_of_the_moon(now));
	if (crm_element_value(cron_spec, "moon") != NULL) {
	pcmk__config_warn("Support for 'moon' in date_spec elements "
	"(such as %s) is deprecated and will be removed "
	"in a future release of Pacemaker", ID(cron_spec));
	}

	/* If we get here, either no fields were specified (which is success), or all
	* the fields that were specified had their conditions met (which is also a
	* success). Thus, the result is success.
	*/
	return pcmk_rc_ok;
	}

	static void
	update_field(crm_time_t t, const xmlNode xml, const char *attr,
	void (time_fn)(crm_time_t , int))
	{
	long long value;

	if ((pcmk__scan_ll(crm_element_value(xml, attr), &value, 0LL) == pcmk_rc_ok)
	&& (value != 0LL) && (value >= INT_MIN) && (value <= INT_MAX)) {
	time_fn(t, (int) value);
	}
	}

	static crm_time_t *
	parse_xml_duration(const crm_time_t start, const xmlNode duration_spec)
	{
	crm_time_t *end = pcmk_copy_time(start);

	update_field(end, duration_spec, "years", crm_time_add_years);
	update_field(end, duration_spec, "months", crm_time_add_months);
	update_field(end, duration_spec, "weeks", crm_time_add_weeks);
	update_field(end, duration_spec, "days", crm_time_add_days);
	update_field(end, duration_spec, "hours", crm_time_add_hours);
	update_field(end, duration_spec, "minutes", crm_time_add_minutes);
	update_field(end, duration_spec, "seconds", crm_time_add_seconds);

	return end;
	}

	// Set next_change to t if t is earlier
	static void
	crm_time_set_if_earlier(crm_time_t next_change, crm_time_t t)
	{
	if ((next_change != NULL) && (t != NULL)) {
	if (!crm_time_is_defined(next_change)
	\|\| (crm_time_compare(t, next_change) < 0)) {
	crm_time_set(next_change, t);
	}
	}
	}

	// Information about a block of nvpair elements
	typedef struct sorted_set_s {
	int score; // This block's score for sorting
	const char *name; // This block's ID
	const char *special_name; // ID that should sort first
	xmlNode *attr_set; // This block
	gboolean overwrite; // Whether existing values will be overwritten
	} sorted_set_t;

	static gint
	sort_pairs(gconstpointer a, gconstpointer b)
	{
	const sorted_set_t *pair_a = a;
	const sorted_set_t *pair_b = b;

	if (a == NULL && b == NULL) {
	return 0;
	} else if (a == NULL) {
	return 1;
	} else if (b == NULL) {
	return -1;
	}

	if (pcmk__str_eq(pair_a->name, pair_a->special_name, pcmk__str_casei)) {
	return -1;

	} else if (pcmk__str_eq(pair_b->name, pair_a->special_name, pcmk__str_casei)) {
	return 1;
	}

	/* If we're overwriting values, we want lowest score first, so the highest
	* score is processed last; if we're not overwriting values, we want highest
	* score first, so nothing else overwrites it.
	*/
	if (pair_a->score < pair_b->score) {
	return pair_a->overwrite? -1 : 1;
	} else if (pair_a->score > pair_b->score) {
	return pair_a->overwrite? 1 : -1;
	}
	return 0;
	}

	static void
	populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlNode * top)
	{
	const char *name = NULL;
	const char *value = NULL;
	const char *old_value = NULL;
	xmlNode *list = nvpair_list;
	xmlNode *an_attr = NULL;

	if (pcmk__xe_is(list->children, XML_TAG_ATTRS)) {
	list = list->children;
	}

	for (an_attr = pcmk__xe_first_child(list); an_attr != NULL;
	an_attr = pcmk__xe_next(an_attr)) {

	if (pcmk__str_eq((const char *)an_attr->name, XML_CIB_TAG_NVPAIR, pcmk__str_none)) {
	xmlNode *ref_nvpair = expand_idref(an_attr, top);

	name = crm_element_value(an_attr, XML_NVPAIR_ATTR_NAME);
	if (name == NULL) {
	name = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_NAME);
	}

	value = crm_element_value(an_attr, XML_NVPAIR_ATTR_VALUE);
	if (value == NULL) {
	value = crm_element_value(ref_nvpair, XML_NVPAIR_ATTR_VALUE);
	}

	if (name == NULL \|\| value == NULL) {
	continue;
	}

	old_value = g_hash_table_lookup(hash, name);

	if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	if (old_value) {
	crm_trace("Letting %s default (removing explicit value \"%s\")",
	name, value);
	g_hash_table_remove(hash, name);
	}
	continue;

	} else if (old_value == NULL) {
	crm_trace("Setting %s=\"%s\"", name, value);
	g_hash_table_insert(hash, strdup(name), strdup(value));

	} else if (overwrite) {
	crm_trace("Setting %s=\"%s\" (overwriting old value \"%s\")",
	name, value, old_value);
	g_hash_table_replace(hash, strdup(name), strdup(value));
	}
	}
	}
	}

	typedef struct unpack_data_s {
	gboolean overwrite;
	void *hash;
	crm_time_t *next_change;
	const pe_rule_eval_data_t *rule_data;
	xmlNode *top;
	} unpack_data_t;

	static void
	unpack_attr_set(gpointer data, gpointer user_data)
	{
	sorted_set_t *pair = data;
	unpack_data_t *unpack_data = user_data;

	if (!pe_eval_rules(pair->attr_set, unpack_data->rule_data,
	unpack_data->next_change)) {
	return;
	}

	crm_trace("Adding attributes from %s (score %d) %s overwrite",
	pair->name, pair->score,
	(unpack_data->overwrite? "with" : "without"));
	populate_hash(pair->attr_set, unpack_data->hash, unpack_data->overwrite, unpack_data->top);
	}

	/*!
	* \internal
	* \brief Create a sorted list of nvpair blocks
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only get blocks of this element
	* \param[in] always_first If not NULL, sort block with this ID as first
	*
	* \return List of sorted_set_t entries for nvpair blocks
	*/
	static GList *
	make_pairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const char *always_first, gboolean overwrite)
	{
	GList *unsorted = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}
	for (xmlNode *attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (pcmk__str_eq(set_name, (const char *) attr_set->name,
	pcmk__str_null_matches)) {
	const char *score = NULL;
	sorted_set_t *pair = NULL;
	xmlNode *expanded_attr_set = expand_idref(attr_set, top);

	if (expanded_attr_set == NULL) {
	// Schema (if not "none") prevents this
	continue;
	}

	pair = calloc(1, sizeof(sorted_set_t));
	pair->name = ID(expanded_attr_set);
	pair->special_name = always_first;
	pair->attr_set = expanded_attr_set;
	pair->overwrite = overwrite;

	score = crm_element_value(expanded_attr_set, XML_RULE_ATTR_SCORE);
	pair->score = char2score(score);

	unsorted = g_list_prepend(unsorted, pair);
	}
	}
	return g_list_sort(unsorted, sort_pairs);
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_eval_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *next_change)
	{
	GList *pairs = make_pairs(top, xml_obj, set_name, always_first, overwrite);

	if (pairs) {
	unpack_data_t data = {
	.hash = hash,
	.overwrite = overwrite,
	.next_change = next_change,
	.top = top,
	.rule_data = rule_data
	};

	g_list_foreach(pairs, unpack_attr_set, &data);
	g_list_free_full(pairs, free);
	}
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name Element name to identify nvpair blocks
	* \param[in] node_hash Node attributes to use when evaluating rules
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in] now Time to use when evaluating rules
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_unpack_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t now, crm_time_t next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash,
	always_first, overwrite, next_change);
	}

	/*!
	* \brief Expand any regular expression submatches (%0-%9) in a string
	*
	* \param[in] string String possibly containing submatch variables
	* \param[in] match_data If not NULL, regular expression matches
	*
	* \return Newly allocated string identical to \p string with submatches
	* expanded, or NULL if there were no matches
	*/
	char *
	pe_expand_re_matches(const char string, const pe_re_match_data_t match_data)
	{
	size_t len = 0;
	int i;
	const char p, last_match_index;
	char p_dst, result = NULL;

	if (pcmk__str_empty(string) \|\| !match_data) {
	return NULL;
	}

	p = last_match_index = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	len += p - last_match_index + (match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so);
	last_match_index = p + 2;
	}
	p++;
	}
	p++;
	}
	len += p - last_match_index + 1;

	/* FIXME: Excessive? */
	if (len - 1 <= 0) {
	return NULL;
	}

	p_dst = result = calloc(1, len);
	p = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	/* rm_eo can be equal to rm_so, but then there is nothing to do */
	int match_len = match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so;
	memcpy(p_dst, match_data->string + match_data->pmatch[i].rm_so, match_len);
	p_dst += match_len;
	}
	p++;
	} else {
	(p_dst) = (p);
	p_dst++;
	}
	p++;
	}

	return result;
	}

	/*!
	* \brief Evaluate rules
	*
	* \param[in,out] ruleset XML possibly containing rule sub-elements
	* \param[in] rule_data
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if there are no rules or
	*/
	gboolean
	pe_eval_rules(xmlNode ruleset, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	// If there are no rules, pass by default
	gboolean ruleset_default = TRUE;

	for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE);
	rule != NULL; rule = crm_next_same_xml(rule)) {

	ruleset_default = FALSE;
	if (pe_eval_expr(rule, rule_data, next_change)) {
	/* Only the deprecated "lifetime" element of location constraints
	* may contain more than one rule at the top level -- the schema
	* limits a block of nvpairs to a single top-level rule. So, this
	* effectively means that a lifetime is active if any rule it
	* contains is active.
	*/
	return TRUE;
	}
	}

	return ruleset_default;
	}

	/*!
	* \brief Evaluate all of a rule's expressions
	*
	* \param[in,out] rule XML containing a rule definition or its id-ref
	* \param[in] rule_data Matching parameters to check against rule
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p rule, otherwise FALSE
	*/
	gboolean
	pe_eval_expr(xmlNode rule, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	xmlNode *expr = NULL;
	gboolean test = TRUE;
	gboolean empty = TRUE;
	gboolean passed = TRUE;
	gboolean do_and = TRUE;
	const char *value = NULL;

	rule = expand_idref(rule, NULL);
	value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP);
	if (pcmk__str_eq(value, "or", pcmk__str_casei)) {
	do_and = FALSE;
	passed = FALSE;
	}

	crm_trace("Testing rule %s", ID(rule));
	for (expr = pcmk__xe_first_child(rule); expr != NULL;
	expr = pcmk__xe_next(expr)) {

	test = pe_eval_subexpr(expr, rule_data, next_change);
	empty = FALSE;

	if (test && do_and == FALSE) {
	crm_trace("Expression %s/%s passed", ID(rule), ID(expr));
	return TRUE;

	} else if (test == FALSE && do_and) {
	crm_trace("Expression %s/%s failed", ID(rule), ID(expr));
	return FALSE;
	}
	}

	if (empty) {
	crm_err("Invalid Rule %s: rules must contain at least one expression", ID(rule));
	}

	crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed");
	return passed;
	}

	/*!
	* \brief Evaluate a single rule expression, including any subexpressions
	*
	* \param[in,out] expr XML containing a rule expression
	* \param[in] rule_data Matching parameters to check against expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p expr, otherwise FALSE
	*/
	gboolean
	pe_eval_subexpr(xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	gboolean accept = FALSE;
	const char *uname = NULL;

	switch (find_expression_type(expr)) {
	case pcmk__subexpr_rule:
	accept = pe_eval_expr(expr, rule_data, next_change);
	break;
	case pcmk__subexpr_attribute:
	case pcmk__subexpr_location:
	/* these expressions can never succeed if there is
	* no node to compare with
	*/
	if (rule_data->node_hash != NULL) {
	accept = pe__eval_attr_expr(expr, rule_data);
	}
	break;

	case pcmk__subexpr_datetime:
	switch (pe__eval_date_expr(expr, rule_data, next_change)) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	accept = TRUE;
	break;

	default:
	accept = FALSE;
	break;
	}
	break;

	case pcmk__subexpr_role:
	accept = pe__eval_role_expr(expr, rule_data);
	break;

	case pcmk__subexpr_resource:
	accept = pe__eval_rsc_expr(expr, rule_data);
	break;

	case pcmk__subexpr_operation:
	accept = pe__eval_op_expr(expr, rule_data);
	break;

	default:
	CRM_CHECK(FALSE /* bad type */ , return FALSE);
	accept = FALSE;
	}
	if (rule_data->node_hash) {
	uname = g_hash_table_lookup(rule_data->node_hash, CRM_ATTR_UNAME);
	}

	crm_trace("Expression %s %s on %s",
	ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes");
	return accept;
	}

	/*!
	* \internal
	* \brief Compare two values in a rule's node attribute expression
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison
	*
	* \return -1 if <tt>(l_val < r_val)</tt>,
	* 0 if <tt>(l_val == r_val)</tt>,
	* 1 if <tt>(l_val > r_val)</tt>
	*/
	static int
	compare_attr_expr_vals(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp = 0;

	if (l_val != NULL && r_val != NULL) {
	if (type == NULL) {
	if (pcmk__strcase_any_of(op, "lt", "lte", "gt", "gte", NULL)) {
	if (pcmk__char_in_any_str('.', l_val, r_val, NULL)) {
	type = "number";
	} else {
	type = "integer";
	}

	} else {
	type = "string";
	}
	crm_trace("Defaulting to %s based comparison for '%s' op", type, op);
	}

	if (pcmk__str_eq(type, "string", pcmk__str_casei)) {
	cmp = strcasecmp(l_val, r_val);

	} else if (pcmk__str_eq(type, "integer", pcmk__str_casei)) {
	long long l_val_num;
	int rc1 = pcmk__scan_ll(l_val, &l_val_num, 0LL);

	long long r_val_num;
	int rc2 = pcmk__scan_ll(r_val, &r_val_num, 0LL);

	if ((rc1 == pcmk_rc_ok) && (rc2 == pcmk_rc_ok)) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Integer parse error. Comparing %s and %s as strings",
	l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "number", pcmk__str_casei)) {
	double l_val_num;
	double r_val_num;

	int rc1 = pcmk__scan_double(l_val, &l_val_num, NULL, NULL);
	int rc2 = pcmk__scan_double(r_val, &r_val_num, NULL, NULL);

	if (rc1 == pcmk_rc_ok && rc2 == pcmk_rc_ok) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Floating-point parse error. Comparing %s and %s as "
	"strings", l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "version", pcmk__str_casei)) {
	cmp = compare_version(l_val, r_val);

	}

	} else if (l_val == NULL && r_val == NULL) {
	cmp = 0;
	} else if (r_val == NULL) {
	cmp = 1;
	} else { // l_val == NULL && r_val != NULL
	cmp = -1;
	}

	return cmp;
	}

	/*!
	* \internal
	* \brief Check whether an attribute expression evaluates to \c true
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison.
	*
	* \return \c true if expression evaluates to \c true, \c false
	* otherwise
	*/
	static bool
	accept_attr_expr(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp;

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	return (l_val != NULL);

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	return (l_val == NULL);

	}

	cmp = compare_attr_expr_vals(l_val, r_val, type, op);

	if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	return (cmp == 0);

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)) {
	return (cmp != 0);

	} else if (l_val == NULL \|\| r_val == NULL) {
	// The comparison is meaningless from this point on
	return false;

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	return (cmp < 0);

	} else if (pcmk__str_eq(op, "lte", pcmk__str_casei)) {
	return (cmp <= 0);

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	return (cmp > 0);

	} else if (pcmk__str_eq(op, "gte", pcmk__str_casei)) {
	return (cmp >= 0);
	}

	return false; // Should never reach this point
	}

	/*!
	* \internal
	* \brief Get correct value according to value-source
	*
	* \param[in] value value given in rule expression
	* \param[in] value_source value-source given in rule expressions
	* \param[in] match_data If not NULL, resource back-references and params
	*/
	static const char *
	expand_value_source(const char value, const char value_source,
	const pe_match_data_t *match_data)
	{
	GHashTable *table = NULL;

	if (pcmk__str_empty(value)) {
	return NULL; // value_source is irrelevant

	} else if (pcmk__str_eq(value_source, "param", pcmk__str_casei)) {
	table = match_data->params;

	} else if (pcmk__str_eq(value_source, "meta", pcmk__str_casei)) {
	table = match_data->meta;

	} else { // literal
	return value;
	}

	if (table == NULL) {
	return NULL;
	}
	return (const char *) g_hash_table_lookup(table, value);
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #uname, #id, #kind,
	* or a generic node attribute
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data The match_data and node_hash members are used
	*
	* \return TRUE if rule_data satisfies the expression, FALSE otherwise
	*/
	gboolean
	pe__eval_attr_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	gboolean attr_allocated = FALSE;
	const char *h_val = NULL;

	const char *op = NULL;
	const char *type = NULL;
	const char *attr = NULL;
	const char *value = NULL;
	const char *value_source = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);
	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);
	value = crm_element_value(expr, XML_EXPR_ATTR_VALUE);
	type = crm_element_value(expr, XML_EXPR_ATTR_TYPE);
	value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE);

	if (attr == NULL) {
	- pe_err("Expression %s invalid: " XML_EXPR_ATTR_ATTRIBUTE
	- " not specified", pcmk__s(ID(expr), "without ID"));
	+ pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_ATTRIBUTE
	+ " not specified", pcmk__s(ID(expr), "without ID"));
	return FALSE;
	} else if (op == NULL) {
	- pe_err("Expression %s invalid: " XML_EXPR_ATTR_OPERATION
	- " not specified", pcmk__s(ID(expr), "without ID"));
	+ pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_OPERATION
	+ " not specified", pcmk__s(ID(expr), "without ID"));
	}

	if (rule_data->match_data != NULL) {
	// Expand any regular expression submatches (%0-%9) in attribute name
	if (rule_data->match_data->re != NULL) {
	char *resolved_attr = pe_expand_re_matches(attr, rule_data->match_data->re);

	if (resolved_attr != NULL) {
	attr = (const char *) resolved_attr;
	attr_allocated = TRUE;
	}
	}

	// Get value appropriate to value-source
	value = expand_value_source(value, value_source, rule_data->match_data);
	}

	if (rule_data->node_hash != NULL) {
	h_val = (const char *)g_hash_table_lookup(rule_data->node_hash, attr);
	}

	if (attr_allocated) {
	free((char *)attr);
	attr = NULL;
	}

	return accept_attr_expr(h_val, value, type, op);
	}

	/*!
	* \internal
	* \brief Evaluate a date_expression
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the now member is used
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return Standard Pacemaker return code
	*/
	int
	pe__eval_date_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	crm_time_t *start = NULL;
	crm_time_t *end = NULL;
	const char *value = NULL;
	const char *op = crm_element_value(expr, "operation");

	xmlNode *duration_spec = NULL;
	xmlNode *date_spec = NULL;

	// "undetermined" will also be returned for parsing errors
	int rc = pcmk_rc_undetermined;

	crm_trace("Testing expression: %s", ID(expr));

	duration_spec = first_named_child(expr, "duration");
	date_spec = first_named_child(expr, "date_spec");

	value = crm_element_value(expr, "start");
	if (value != NULL) {
	start = crm_time_new(value);
	}
	value = crm_element_value(expr, "end");
	if (value != NULL) {
	end = crm_time_new(value);
	}

	if (start != NULL && end == NULL && duration_spec != NULL) {
	end = parse_xml_duration(start, duration_spec);
	}

	if (pcmk__str_eq(op, "in_range", pcmk__str_null_matches \| pcmk__str_casei)) {
	if ((start == NULL) && (end == NULL)) {
	// in_range requires at least one of start or end
	} else if ((start != NULL) && (crm_time_compare(rule_data->now, start) < 0)) {
	rc = pcmk_rc_before_range;
	crm_time_set_if_earlier(next_change, start);
	} else if ((end != NULL) && (crm_time_compare(rule_data->now, end) > 0)) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	if (end && next_change) {
	// Evaluation doesn't change until second after end
	crm_time_add_seconds(end, 1);
	crm_time_set_if_earlier(next_change, end);
	}
	}

	} else if (pcmk__str_eq(op, "date_spec", pcmk__str_casei)) {
	rc = pe_cron_range_satisfied(rule_data->now, date_spec);
	// @TODO set next_change appropriately

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	if (start == NULL) {
	// gt requires start
	} else if (crm_time_compare(rule_data->now, start) > 0) {
	rc = pcmk_rc_within_range;
	} else {
	rc = pcmk_rc_before_range;

	// Evaluation doesn't change until second after start
	crm_time_add_seconds(start, 1);
	crm_time_set_if_earlier(next_change, start);
	}

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	if (end == NULL) {
	// lt requires end
	} else if (crm_time_compare(rule_data->now, end) < 0) {
	rc = pcmk_rc_within_range;
	crm_time_set_if_earlier(next_change, end);
	} else {
	rc = pcmk_rc_after_range;
	}
	}

	crm_time_free(start);
	crm_time_free(end);
	return rc;
	}

	gboolean
	pe__eval_op_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *name = crm_element_value(expr, XML_NVPAIR_ATTR_NAME);
	const char *interval_s = crm_element_value(expr, XML_LRM_ATTR_INTERVAL);
	guint interval;

	crm_trace("Testing op_defaults expression: %s", ID(expr));

	if (rule_data->op_data == NULL) {
	crm_trace("No operations data provided");
	return FALSE;
	}

	interval = crm_parse_interval_spec(interval_s);
	if (interval == 0 && errno != 0) {
	crm_trace("Could not parse interval: %s", interval_s);
	return FALSE;
	}

	if (interval_s != NULL && interval != rule_data->op_data->interval) {
	crm_trace("Interval doesn't match: %d != %d", interval, rule_data->op_data->interval);
	return FALSE;
	}

	if (!pcmk__str_eq(name, rule_data->op_data->op_name, pcmk__str_none)) {
	crm_trace("Name doesn't match: %s != %s", name, rule_data->op_data->op_name);
	return FALSE;
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #role
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the role member is used
	*
	* \return TRUE if rule_data->role satisfies the expression, FALSE otherwise
	*/
	gboolean
	pe__eval_role_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	gboolean accept = FALSE;
	const char *op = NULL;
	const char *value = NULL;

	if (rule_data->role == pcmk_role_unknown) {
	return accept;
	}

	value = crm_element_value(expr, XML_EXPR_ATTR_VALUE);
	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	if (rule_data->role > pcmk_role_started) {
	accept = TRUE;
	}

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	if ((rule_data->role > pcmk_role_unknown)
	&& (rule_data->role < pcmk_role_unpromoted)) {
	accept = TRUE;
	}

	} else if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	if (text2role(value) == rule_data->role) {
	accept = TRUE;
	}

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)) {
	// Test "ne" only with promotable clone roles
	if ((rule_data->role > pcmk_role_unknown)
	&& (rule_data->role < pcmk_role_unpromoted)) {
	accept = FALSE;

	} else if (text2role(value) != rule_data->role) {
	accept = TRUE;
	}
	}
	return accept;
	}

	gboolean
	pe__eval_rsc_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *class = crm_element_value(expr, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(expr, XML_AGENT_ATTR_PROVIDER);
	const char *type = crm_element_value(expr, XML_EXPR_ATTR_TYPE);

	crm_trace("Testing rsc_defaults expression: %s", ID(expr));

	if (rule_data->rsc_data == NULL) {
	crm_trace("No resource data provided");
	return FALSE;
	}

	if (class != NULL &&
	!pcmk__str_eq(class, rule_data->rsc_data->standard, pcmk__str_none)) {
	crm_trace("Class doesn't match: %s != %s", class, rule_data->rsc_data->standard);
	return FALSE;
	}

	if ((provider == NULL && rule_data->rsc_data->provider != NULL) \|\|
	(provider != NULL && rule_data->rsc_data->provider == NULL) \|\|
	!pcmk__str_eq(provider, rule_data->rsc_data->provider, pcmk__str_none)) {
	crm_trace("Provider doesn't match: %s != %s", provider, rule_data->rsc_data->provider);
	return FALSE;
	}

	if (type != NULL &&
	!pcmk__str_eq(type, rule_data->rsc_data->agent, pcmk__str_none)) {
	crm_trace("Agent doesn't match: %s != %s", type, rule_data->rsc_data->agent);
	return FALSE;
	}

	return TRUE;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/rules_compat.h>

	gboolean
	test_ruleset(xmlNode ruleset, GHashTable node_hash, crm_time_t *now)
	{
	return pe_evaluate_rules(ruleset, node_hash, now, NULL);
	}

	gboolean
	test_rule(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_rule_re(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_rule(rule, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_rule_full(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, pe_match_data_t match_data)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, match_data);
	}

	gboolean
	test_expression(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_expression_re(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_expression(expr, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_expression_full(xmlNode expr, GHashTable node_hash,
	enum rsc_role_e role, crm_time_t *now,
	pe_match_data_t *match_data)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, match_data);
	}

	void
	unpack_instance_attributes(xmlNode top, xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *now)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash, always_first,
	overwrite, NULL);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 79500c87d2..b7cbc8b822 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5089 +1,5090 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	static gboolean
	is_dangling_guest_node(pcmk_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider `priority-fencing-delay`
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pe__node_name(node), reason, rsc->id);
	} else {
	crm_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pe__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pe__node_name(node), reason);
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pe__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	crm_warn("Remote node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply `priority-fencing-delay` for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	crm_warn("Cluster node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
	"and @" XML_NVPAIR_ATTR_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pe__set_working_set_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, scheduler);

	verify_pe_options(scheduler->config_hash);

	set_config_flag(scheduler, "enable-startup-probes",
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(scheduler->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and stonith-watchdog-timeout is nonzero");
	pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pe_pref(scheduler->config_hash, "stonith-timeout");
	scheduler->stonith_timeout = (int) crm_parse_interval_spec(value);
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, "stonith-enabled", pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pe_pref(scheduler->config_hash,
	"stonith-action");
	if (!strcmp(scheduler->stonith_action, "poweroff")) {
	pe_warn_once(pcmk__wo_poweroff,
	"Support for stonith-action of 'poweroff' is deprecated "
	"and will be removed in a future release (use 'off' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, "concurrent-fencing",
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
	if (value) {
	scheduler->priority_fencing_delay = crm_parse_interval_spec(value)
	/ 1000;
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, "stop-all-resources", pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)));

	set_config_flag(scheduler, "symmetric-cluster",
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(scheduler->config_hash, "no-quorum-policy");

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, XML_ATTR_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
	"fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, "stop-orphan-resources",
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, "stop-orphan-actions",
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(scheduler->config_hash, "remove-after-stop");
	if (value != NULL) {
	if (crm_is_true(value)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pe_warn_once(pcmk__wo_remove_after,
	"Support for the remove-after-stop cluster property is"
	" deprecated and will be removed in a future release");
	#endif
	} else {
	pe__clear_working_set_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, "maintenance-mode", pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags,
	pcmk_sched_in_maintenance)));

	set_config_flag(scheduler, "start-failure-is-fatal",
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, "startup-fencing",
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pe_warn_once(pcmk__wo_blind, "Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy = pe_pref(scheduler->config_hash,
	"placement-strategy");
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, "shutdown-lock", pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
	scheduler->shutdown_lock = crm_parse_interval_spec(value) / 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(scheduler->config_hash,
	XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT);
	scheduler->node_pending_timeout = crm_parse_interval_spec(value) / 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pe_find_node(scheduler->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = scheduler;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	pe__set_working_set_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pe_warn_once(pcmk__wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (pcmk__str_eq(name, XML_RSC_ATTR_REMOTE_NODE, pcmk__str_casei)) {
	remote_name = value;
	} else if (pcmk__str_eq(name, "remote-addr", pcmk__str_casei)) {
	remote_server = value;
	} else if (pcmk__str_eq(name, "remote-port", pcmk__str_casei)) {
	remote_port = value;
	} else if (pcmk__str_eq(name, "remote-connect-timeout", pcmk__str_casei)) {
	connect_timeout = value;
	} else if (pcmk__str_eq(name, "remote-allow-migrate", pcmk__str_casei)) {
	remote_allow_migrate=value;
	} else if (pcmk__str_eq(name, XML_RSC_ATTR_MANAGED, pcmk__str_casei)) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pe_find_node(scheduler->nodes, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	setup_container(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, scheduler);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pcmk_resource_t *container = pe_find_resource(scheduler->resources,
	container_id);

	if (container) {
	rsc->container = container;
	pe__set_resource_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's container is %s",
	rsc->id, container_id);
	} else {
	- pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
	+ pcmk__config_err("Resource %s: Unknown resource container (%s)",
	+ rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(scheduler->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pe__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(scheduler->template_rsc_sets, strdup(id),
	NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " XML_ATTR_ID "='%s'>",
	xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	setup_container(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, XML_ATTR_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pe__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pe__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pe__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pe__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pe__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute for %s because disabling resource discovery "
	"is not allowed for cluster nodes", pe__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	if (id == NULL) {
	crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
	XML_ATTR_ID);
	return;
	}

	uname = crm_element_value(state, XML_ATTR_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created node_state entry doesn't have an uname
	* yet. We should recognize the node as `pending` and wait for it to
	* join CPG.
	*/
	crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	XML_ATTR_UNAME, id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pe__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, XML_ATTR_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	* \param[in] node_state Node's node_state entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for node-pending-timeout.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	" in " XML_CIB_TAG_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	* \param[in] node_state Node's node_state entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK__XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, OFFLINESTATUS,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, ONLINESTATUS, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_CRMD " in "
	XML_CIB_TAG_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	* \param[in] node_state Node's node_state entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pe_node_attribute_raw(node, PCMK_NODE_ATTR_TERMINATE);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pe__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pe__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pe__node_name(this_node), join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* node-pending-timeout, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->details->data_set) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK__XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK__XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK__XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pe__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pe__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pe__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider `priority-fencing-delay` for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pe__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->details->remote_rsc;
	pcmk_resource_t *container = NULL;
	pcmk_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pe__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pe__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pe__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pe__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(scheduler->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pe__set_resource_flags(rsc, pcmk_rsc_removed_filler);
	}
	pe__set_resource_flags(rsc, pcmk_rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);

	// find_rsc() because we might be a cloned group
	pcmk_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pe__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of clone-max instances);
	* (3) a newly created orphan (i.e. clone-max instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pe__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if globally-unique
	* was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pcmk_node_t *) locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if globally-unique is switched from true to
	* false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pe__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when clone-max=0, we create
	* a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pe__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__",
	scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pe__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pe__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pe__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pe__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pe__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pe__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pe__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pe__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pe__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pe__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pcmk_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	crm_warn("Ignoring malformed " XML_LRM_TAG_RESOURCE
	" entry without id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pe__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, XML_RSC_ATTR_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(scheduler->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, XML_LRM_ATTR_TASK);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	crm_err("Ignoring resource history entry %s without "
	XML_LRM_ATTR_MIGRATE_SOURCE " and " XML_LRM_ATTR_MIGRATE_TARGET,
	ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	crm_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	ID(entry), *source_name, pe__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	crm_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	ID(entry), *target_name, pe__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pe__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
	&from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pe__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pe__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -INFINITY;
	pcmk_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	crm_warn("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	history->rsc->id, pe__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	fail2text(*on_fail), fail2text(config_on_fail),
	history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(config_on_fail), role2text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pe__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pe__clear_resource_flags(history->rsc, pcmk_rsc_managed);
	pe__set_resource_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	crm_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pe__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pe__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pe__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pe__xe_history_key(xml_op),
	node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pe_find_node(scheduler->nodes, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pe__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	// Resource has a failure-timeout, and history entry has a timestamp

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pe__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pe__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pe__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	crm_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	history->id, history->rsc->id, pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_OPSTATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	crm_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_RC " '%s'",
	history->id, history->rsc->id, pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_RC),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml,
	XML_LRM_ATTR_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pe__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, XML_LRM_ATTR_TASK),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	XML_RSC_OP_LAST_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pe__set_resource_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	XML_LRM_ATTR_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	crm_err("Ignoring resource history entry for %s on %s without ID",
	rsc->id, pe__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (history.task == NULL) {
	crm_err("Ignoring resource history entry %s for %s on %s without "
	XML_LRM_ATTR_TASK, history.id, rsc->id, pe__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
	&(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pe__node_name(node), history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pe__xe_history_key(xml_op);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pe__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pe__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pe__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
	last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	pe__set_resource_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pe__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	crm_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " %s",
	parent->id, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pe__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, scheduler);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(scheduler->input, XML_CIB_TAG_STATUS, TRUE);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(scheduler->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 2:21 PM (1 d, 16 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1659526
Default Alt Text: (690 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions