No OneTemporary
Actions

Size

188 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index d0f8322ee3..33635eb436 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,615 +1,618 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H

	# include <stdint.h>
	# include <string.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>

	enum pe__clone_flags {
	// Whether instances should be started sequentially
	pe__clone_ordered = (1 << 0),

	// Whether promotion scores have been added
	pe__clone_promotion_added = (1 << 1),

	// Whether promotion constraints have been added
	pe__clone_promotion_constrained = (1 << 2),
	};

	bool pe__clone_is_ordered(pe_resource_t *clone);
	int pe__set_clone_flag(pe_resource_t *clone, enum pe__clone_flags flag);


	# define pe_rsc_info(rsc, fmt, args...) crm_log_tag(LOG_INFO, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)

	# define pe_err(fmt...) do { \
	was_processing_error = TRUE; \
	pcmk__config_err(fmt); \
	} while (0)

	# define pe_warn(fmt...) do { \
	was_processing_warning = TRUE; \
	pcmk__config_warn(fmt); \
	} while (0)

	# define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
	# define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }

	#define pe__set_working_set_flags(working_set, flags_to_set) do { \
	(working_set)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(working_set, flags_to_clear) do { \
	(working_set)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_graph_flags(graph_flags, gr_action, flags_to_set) do { \
	graph_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_graph_flags(graph_flags, gr_action, flags_to_clear) do { \
	graph_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Some warnings we don't want to print every transition

	enum pe_warn_once_e {
	pe_wo_blind = (1 << 0),
	pe_wo_restart_type = (1 << 1),
	pe_wo_role_after = (1 << 2),
	pe_wo_poweroff = (1 << 3),
	pe_wo_require_all = (1 << 4),
	pe_wo_order_score = (1 << 5),
	pe_wo_neg_threshold = (1 << 6),
	pe_wo_remove_after = (1 << 7),
	pe_wo_ping_node = (1 << 8),
	};

	extern uint32_t pe_wo;

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pe_wo, pe_wo_bit)) { \
	if (pe_wo_bit == pe_wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pe_warn(fmt); \
	} \
	pe_wo = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Warn-once", "logging", pe_wo, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);


	typedef struct pe__location_constraint_s {
	char *id; // Constraint XML ID
	pe_resource_t *rsc_lh; // Resource being located
	enum rsc_role_e role_filter; // Role to locate
	enum pe_discover_e discover_mode; // Resource discovery
	GList node_list_rh; // List of pe_node_t
	} pe__location_t;

	typedef struct pe__order_constraint_s {
	int id;
	enum pe_ordering type;

	void *lh_opaque;
	pe_resource_t *lh_rsc;
	pe_action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	pe_resource_t *rh_rsc;
	pe_action_t *rh_action;
	char *rh_action_task;
	} pe__ordering_t;

	typedef struct notify_data_s {
	GSList *keys; // Environment variable name/value pairs

	const char *action;

	pe_action_t *pre;
	pe_action_t *post;
	pe_action_t *pre_done;
	pe_action_t *post_done;

	GList active; / notify_entry_t* */
	GList inactive; / notify_entry_t* */
	GList start; / notify_entry_t* */
	GList stop; / notify_entry_t* */
	GList demote; / notify_entry_t* */
	GList promote; / notify_entry_t* */
	GList promoted; / notify_entry_t* */
	GList unpromoted; / notify_entry_t* */
	GHashTable *allowed_nodes;

	} notify_data_t;

	+int pe__clone_promoted_max(pe_resource_t *clone);
	+int pe__clone_promoted_node_max(pe_resource_t *clone);
	+
	bool pe_can_fence(pe_working_set_t data_set, pe_node_t node);

	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pe_resource_t rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set);
	pe_node_t native_location(const pe_resource_t rsc, GList **list, int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void common_update_score(pe_resource_t * rsc, const char *id, int score);
	void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed);

	gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set);

	pe_resource_t native_find_rsc(pe_resource_t rsc, const char id, const pe_node_t node,
	int flags);

	gboolean native_active(pe_resource_t * rsc, gboolean all);
	gboolean group_active(pe_resource_t * rsc, gboolean all);
	gboolean clone_active(pe_resource_t * rsc, gboolean all);
	gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);

	void native_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void group_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void clone_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar * pcmk__native_output_string(pe_resource_t rsc, const char name, pe_node_t *node,
	uint32_t show_opts, const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pe_node_t node, bool print_detail);

	static inline const char *
	pe__rsc_bool_str(pe_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pe_resource_t * rsc);
	void group_free(pe_resource_t * rsc);
	void clone_free(pe_resource_t * rsc);
	void pe__free_bundle(pe_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
	gboolean current);

	void pe__count_common(pe_resource_t *rsc);
	void pe__count_bundle(pe_resource_t *rsc);

	gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
	pe_working_set_t * data_set);
	void common_free(pe_resource_t * rsc);

	pe_node_t pe__copy_node(const pe_node_t this_node);
	extern time_t get_effective_time(pe_working_set_t * data_set);

	/* Failure handling utilities (from failcounts.c) */

	// bit flags for fail count handling options
	enum pe_fc_flags_e {
	pe_fc_default = (1 << 0),
	pe_fc_effective = (1 << 1), // don't count expired failures
	pe_fc_fillers = (1 << 2), // if container, include filler failures in count
	};

	int pe_get_failcount(pe_node_t node, pe_resource_t rsc, time_t *last_failure,
	uint32_t flags, xmlNode *xml_op,
	pe_working_set_t *data_set);

	pe_action_t pe__clear_failcount(pe_resource_t rsc, pe_node_t *node,
	const char *reason,
	pe_working_set_t *data_set);

	/* Functions for finding/counting a resource's active nodes */

	pe_node_t pe__find_active_on(const pe_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);
	pe_node_t pe__find_active_requires(const pe_resource_t rsc,
	unsigned int *count);

	static inline pe_node_t *
	pe__current_node(const pe_resource_t *rsc)
	{
	return pe__find_active_on(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	extern void node_list_exclude(GHashTable * list, GList *list2, gboolean merge_scores);

	GHashTable pe__node_list2table(GList list);

	static inline gpointer
	pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
	{
	if (hash) {
	return g_hash_table_lookup(hash, key);
	}
	return NULL;
	}

	extern pe_action_t get_pseudo_op(const char name, pe_working_set_t * data_set);
	extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);

	/* Printing functions for debug */
	extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
	extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);

	void pe__show_node_weights_as(const char file, const char function,
	int line, bool to_log, pe_resource_t *rsc,
	const char comment, GHashTable nodes,
	pe_working_set_t *data_set);

	#define pe__show_node_weights(level, rsc, text, nodes, data_set) \
	pe__show_node_weights_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (data_set))

	extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);

	extern xmlNode find_rsc_op_entry(pe_resource_t rsc, const char *key);

	extern pe_action_t custom_action(pe_resource_t rsc, char key, const char task, pe_node_t * on_node,
	gboolean optional, gboolean foo, pe_working_set_t * data_set);

	# define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \
	optional, TRUE, data_set);

	# define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
	# define stopped_action(rsc, node, optional) custom_action( \
	rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \
	optional, TRUE, data_set);

	# define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \
	optional, TRUE, data_set);

	# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), CRMD_ACTION_START, node, \
	optional, TRUE, data_set)

	# define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
	# define started_action(rsc, node, optional) custom_action( \
	rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \
	optional, TRUE, data_set)

	# define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \
	optional, TRUE, data_set)

	# define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
	# define promoted_action(rsc, node, optional) custom_action( \
	rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \
	optional, TRUE, data_set)

	# define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \
	optional, TRUE, data_set)

	# define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
	# define demoted_action(rsc, node, optional) custom_action( \
	rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \
	optional, TRUE, data_set)

	extern int pe_get_configured_timeout(pe_resource_t rsc, const char action,
	pe_working_set_t *data_set);

	extern pe_action_t find_first_action(GList input, const char uuid, const char task,
	pe_node_t * on_node);
	extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
	gboolean allow_non_atomic);

	extern GList find_actions(GList input, const char key, const pe_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pe_node_t *on_node);
	extern GList find_recurring_actions(GList input, pe_node_t * not_on_node);
	GList pe__resource_actions(const pe_resource_t rsc, const pe_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pe_action_t * action);

	extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set);

	extern int pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default);
	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);
	void pe__set_next_role(pe_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	extern pe_resource_t find_clone_instance(pe_resource_t rsc, const char *sub_id,
	pe_working_set_t * data_set);

	extern void destroy_ticket(gpointer data);
	extern pe_ticket_t ticket_new(const char ticket_id, pe_working_set_t * data_set);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(pe_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return false;
	}

	int pe__target_rc_from_xml(xmlNode *xml_op);

	gint sort_node_uname(gconstpointer a, gconstpointer b);
	bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);

	enum rsc_digest_cmp_val {
	/! Digests are the same /
	RSC_DIGEST_MATCH = 0,
	/! Params that require a restart changed /
	RSC_DIGEST_RESTART,
	/! Some parameter changed. /
	RSC_DIGEST_ALL,
	/*! rsc op didn't have a digest associated with it, so
	* it is unknown if parameters changed or not. */
	RSC_DIGEST_UNKNOWN,
	};

	typedef struct op_digest_cache_s {
	enum rsc_digest_cmp_val rc;
	xmlNode *params_all;
	xmlNode *params_secure;
	xmlNode *params_restart;
	char *digest_all_calc;
	char *digest_secure_calc;
	char *digest_restart_calc;
	} op_digest_cache_t;

	op_digest_cache_t pe__calculate_digests(pe_resource_t rsc, const char *task,
	guint interval_ms, pe_node_t node,
	xmlNode xml_op, GHashTable overrides,
	bool calc_secure,
	pe_working_set_t *data_set);

	void pe__free_digests(gpointer ptr);

	op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc, xmlNode * xml_op, pe_node_t * node,
	pe_working_set_t * data_set);

	pe_action_t pe_fence_op(pe_node_t node, const char op, bool optional, const char reason, bool priority_delay, pe_working_set_t * data_set);
	void trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set);

	char pe__action2reason(pe_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite);

	void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);

	pe_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pe_working_set_t data_set);
	void common_print(pe_resource_t * rsc, const char pre_text, const char name, pe_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	int pe__common_output_html(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, unsigned int options);
	pe_resource_t pe__find_bundle_replica(const pe_resource_t bundle,
	const pe_node_t *node);
	bool pe__bundle_needs_remote_name(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	const char pe__add_bundle_remote_name(pe_resource_t rsc,
	pe_working_set_t *data_set,
	xmlNode xml, const char field);
	const char pe_node_attribute_calculated(const pe_node_t node,
	const char *name,
	const pe_resource_t *rsc);
	const char pe_node_attribute_raw(pe_node_t node, const char *name);
	bool pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	void pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	enum pe_check_parameters, pe_working_set_t *data_set);
	void pe__foreach_param_check(pe_working_set_t *data_set,
	void (cb)(pe_resource_t, pe_node_t, xmlNode,
	enum pe_check_parameters,
	pe_working_set_t*));
	void pe__free_param_checks(pe_working_set_t *data_set);

	bool pe__shutdown_requested(pe_node_t *node);
	void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set);

	bool pe__resource_is_disabled(pe_resource_t *rsc);
	pe_action_t pe__clear_resource_history(pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set);

	GList pe__rscs_with_tag(pe_working_set_t data_set, const char *tag_name);
	GList pe__unames_with_tag(pe_working_set_t data_set, const char *tag_name);
	bool pe__rsc_has_tag(pe_working_set_t data_set, const char rsc, const char *tag);
	bool pe__uname_has_tag(pe_working_set_t data_set, const char node, const char *tag);

	bool pe__rsc_running_on_any(pe_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pe_working_set_t data_set, const char s);
	GList * pe__build_rsc_list(pe_working_set_t data_set, const char s);

	bool pcmk__rsc_filtered_by_node(pe_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__clone_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__group_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);
	gboolean pe__native_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(pe_resource_t rsc, const char *name);

	const char pe__clone_child_id(pe_resource_t rsc);

	int pe__sum_node_health_scores(const pe_node_t *node, int base_health);
	int pe__node_health(pe_node_t *node);

	static inline enum pcmk__health_strategy
	pe__health_strategy(pe_working_set_t *data_set)
	{
	return pcmk__parse_health_strategy(pe_pref(data_set->config_hash,
	PCMK__OPT_NODE_HEALTH_STRATEGY));
	}

	static inline int
	pe__health_score(const char option, pe_working_set_t data_set)
	{
	return char2score(pe_pref(data_set->config_hash, option));
	}

	#endif
	diff --git a/include/pcmki/pcmki_sched_allocate.h b/include/pcmki/pcmki_sched_allocate.h
	index caa747d6e7..bc8d389961 100644
	--- a/include/pcmki/pcmki_sched_allocate.h
	+++ b/include/pcmki/pcmki_sched_allocate.h
	@@ -1,222 +1,220 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__PCMKI_PCMKI_SCHED_ALLOCATE__H
	# define PCMK__PCMKI_PCMKI_SCHED_ALLOCATE__H

	# include <glib.h>
	# include <crm/common/xml.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/complex.h>
	# include <crm/common/xml_internal.h>
	# include <crm/pengine/internal.h>
	# include <crm/common/xml.h>
	# include <pcmki/pcmki_scheduler.h>

	struct resource_alloc_functions_s {
	GHashTable (merge_weights) (pe_resource_t , const char , GHashTable , const char , float,
	enum pe_weights);
	pe_node_t (allocate) (pe_resource_t , pe_node_t , pe_working_set_t *);
	void (create_actions) (pe_resource_t , pe_working_set_t *);
	gboolean(create_probe) (pe_resource_t , pe_node_t , pe_action_t , gboolean, pe_working_set_t *);
	void (internal_constraints) (pe_resource_t , pe_working_set_t *);

	void (rsc_colocation_lh) (pe_resource_t , pe_resource_t *,
	pcmk__colocation_t , pe_working_set_t );
	void (rsc_colocation_rh) (pe_resource_t , pe_resource_t *,
	pcmk__colocation_t , pe_working_set_t );

	/*!
	* \internal
	* \brief Create list of all resources in colocations with a given resource
	*
	* Given a resource, create a list of all resources involved in mandatory
	* colocations with it, whether directly or indirectly via chained colocations.
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in] colocated_rscs Existing list
	*
	* \return List of given resource and all resources involved in colocations
	*
	* \note This function is recursive; top-level callers should pass NULL as
	* \p colocated_rscs and \p orig_rsc, and the desired resource as
	* \p rsc. The recursive calls will use other values.
	*/
	GList (colocated_resources)(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList *colocated_rscs);

	void (rsc_location) (pe_resource_t , pe__location_t *);

	enum pe_action_flags (action_flags) (pe_action_t , pe_node_t *);
	enum pe_graph_flags (update_actions) (pe_action_t , pe_action_t *,
	pe_node_t *, enum pe_action_flags,
	enum pe_action_flags,
	enum pe_ordering,
	pe_working_set_t *data_set);
	void (output_actions)(pe_resource_t rsc);

	void (expand) (pe_resource_t , pe_working_set_t *);
	void (append_meta) (pe_resource_t rsc, xmlNode * xml);

	/*!
	* \internal
	* \brief Add a resource's utilization to a table of utilization values
	*
	* This function is used when summing the utilization of a resource and all
	* resources colocated with it, to determine whether a node has sufficient
	* capacity. Given a resource and a table of utilization values, it will add
	* the resource's utilization to the existing values, if the resource has
	* not yet been allocated to a node.
	*
	* \param[in] rsc Resource with utilization to add
	* \param[in] orig_rsc Resource being allocated (for logging only)
	* \param[in] all_rscs List of all resources that will be summed
	* \param[in] utilization Table of utilization values to add to
	*/
	void (add_utilization)(pe_resource_t rsc, pe_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization);

	/*!
	* \internal
	* \brief Apply a shutdown lock for a resource, if appropriate
	*
	* \param[in] rsc Resource to check for shutdown lock
	*/
	void (shutdown_lock)(pe_resource_t rsc);
	};

	GHashTable pcmk__native_merge_weights(pe_resource_t rsc, const char *rhs,
	GHashTable nodes, const char attr,
	float factor, uint32_t flags);

	GHashTable pcmk__group_merge_weights(pe_resource_t rsc, const char *rhs,
	GHashTable nodes, const char attr,
	float factor, uint32_t flags);

	pe_node_t pcmk__native_allocate(pe_resource_t rsc, pe_node_t *preferred,
	pe_working_set_t *data_set);
	extern void native_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern void native_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
	void native_rsc_colocation_lh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void native_rsc_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	extern enum pe_action_flags native_action_flags(pe_action_t * action, pe_node_t * node);

	void native_rsc_location(pe_resource_t rsc, pe__location_t constraint);
	extern void native_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern gboolean native_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set);
	extern void native_append_meta(pe_resource_t * rsc, xmlNode * xml);
	void pcmk__primitive_add_utilization(pe_resource_t *rsc,
	pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization);
	void pcmk__primitive_shutdown_lock(pe_resource_t *rsc);

	pe_node_t pcmk__group_allocate(pe_resource_t rsc, pe_node_t *preferred,
	pe_working_set_t *data_set);
	extern void group_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern void group_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
	void group_rsc_colocation_lh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void group_rsc_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	extern enum pe_action_flags group_action_flags(pe_action_t * action, pe_node_t * node);
	void group_rsc_location(pe_resource_t rsc, pe__location_t constraint);
	extern void group_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern void group_append_meta(pe_resource_t * rsc, xmlNode * xml);
	void pcmk__group_add_utilization(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList all_rscs, GHashTable utilization);
	void pcmk__group_shutdown_lock(pe_resource_t *rsc);

	pe_node_t pcmk__bundle_allocate(pe_resource_t rsc, pe_node_t *preferred,
	pe_working_set_t *data_set);
	void pcmk__bundle_create_actions(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	gboolean pcmk__bundle_create_probe(pe_resource_t rsc, pe_node_t node,
	pe_action_t *complete, gboolean force,
	pe_working_set_t *data_set);
	void pcmk__bundle_internal_constraints(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	void pcmk__bundle_rsc_colocation_lh(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void pcmk__bundle_rsc_colocation_rh(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void pcmk__bundle_rsc_location(pe_resource_t rsc, pe__location_t constraint);
	enum pe_action_flags pcmk__bundle_action_flags(pe_action_t *action,
	pe_node_t *node);
	void pcmk__bundle_expand(pe_resource_t rsc, pe_working_set_t data_set);
	void pcmk__bundle_append_meta(pe_resource_t rsc, xmlNode xml);
	void pcmk__bundle_add_utilization(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList all_rscs, GHashTable utilization);
	void pcmk__bundle_shutdown_lock(pe_resource_t *rsc);

	pe_node_t pcmk__clone_allocate(pe_resource_t rsc, pe_node_t *preferred,
	pe_working_set_t *data_set);
	extern void clone_create_actions(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern void clone_internal_constraints(pe_resource_t * rsc, pe_working_set_t * data_set);
	void clone_rsc_colocation_lh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void clone_rsc_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);
	void clone_rsc_location(pe_resource_t rsc, pe__location_t constraint);
	extern enum pe_action_flags clone_action_flags(pe_action_t * action, pe_node_t * node);
	extern void clone_expand(pe_resource_t * rsc, pe_working_set_t * data_set);
	extern gboolean clone_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set);
	extern void clone_append_meta(pe_resource_t * rsc, xmlNode * xml);
	void pcmk__clone_add_utilization(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList all_rscs, GHashTable utilization);
	void pcmk__clone_shutdown_lock(pe_resource_t *rsc);

	void pcmk__add_promotion_scores(pe_resource_t *rsc);
	-pe_node_t pcmk__set_instance_roles(pe_resource_t rsc,
	- pe_working_set_t *data_set);
	void create_promotable_actions(pe_resource_t rsc, pe_working_set_t data_set);
	void promote_demote_constraints(pe_resource_t rsc, pe_working_set_t data_set);
	void promotable_constraints(pe_resource_t rsc, pe_working_set_t data_set);
	void promotable_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set);

	enum pe_graph_flags native_update_actions(pe_action_t first, pe_action_t then,
	pe_node_t *node,
	enum pe_action_flags flags,
	enum pe_action_flags filter,
	enum pe_ordering type,
	pe_working_set_t *data_set);
	enum pe_graph_flags group_update_actions(pe_action_t first, pe_action_t then,
	pe_node_t *node,
	enum pe_action_flags flags,
	enum pe_action_flags filter,
	enum pe_ordering type,
	pe_working_set_t *data_set);
	enum pe_graph_flags pcmk__multi_update_actions(pe_action_t *first,
	pe_action_t *then,
	pe_node_t *node,
	enum pe_action_flags flags,
	enum pe_action_flags filter,
	enum pe_ordering type,
	pe_working_set_t *data_set);

	void pcmk__log_transition_summary(const char *filename);
	void clone_create_pseudo_actions(
	pe_resource_t * rsc, GList children, notify_data_t start_notify, notify_data_t stop_notify, pe_working_set_t data_set);
	#endif
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index a2fd5baabb..b1a77ccc10 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,435 +1,447 @@
	/*
	* Copyright 2021-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__LIBPACEMAKER_PRIVATE__H
	# define PCMK__LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <crm/pengine/pe_types.h> // pe_action_t, pe_node_t, pe_working_set_t

	// Actions (pcmk_sched_actions.c)

	G_GNUC_INTERNAL
	void pcmk__update_action_for_orderings(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__log_action(const char pre_text, pe_action_t action, bool details);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_rsc_pseudo_action(pe_resource_t rsc, const char *task,
	bool optional, bool runnable);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_cancel_action(pe_resource_t rsc, const char *name,
	guint interval_ms, pe_node_t *node);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_shutdown_action(pe_node_t node);

	G_GNUC_INTERNAL
	bool pcmk__action_locks_rsc_to_node(const pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__deduplicate_action_inputs(pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__output_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__check_action_config(pe_resource_t rsc, pe_node_t node,
	xmlNode *xml_op);

	G_GNUC_INTERNAL
	void pcmk__handle_rsc_config_changes(pe_working_set_t *data_set);

	// Producing transition graphs (pcmk_graph_producer.c)

	G_GNUC_INTERNAL
	bool pcmk__graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input);

	G_GNUC_INTERNAL
	void pcmk__add_action_to_graph(pe_action_t action, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__create_graph(pe_working_set_t *data_set);


	// Fencing (pcmk_sched_fencing.c)

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(pe_resource_t rsc, pe_node_t node,
	pe_action_t *action, enum pe_ordering order,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(pe_node_t *node);


	// Injected scheduler inputs (pcmk_sched_injections.c)

	void pcmk__inject_scheduler_input(pe_working_set_t data_set, cib_t cib,
	pcmk_injections_t *injections);


	// Constraints of any type (pcmk_sched_constraints.c)

	G_GNUC_INTERNAL
	pe_resource_t pcmk__find_constraint_resource(GList rsc_list, const char *id);

	G_GNUC_INTERNAL
	xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	pe_resource_t rsc, pe_tag_t tag);

	G_GNUC_INTERNAL
	bool pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__create_internal_constraints(pe_working_set_t *data_set);


	// Location constraints

	G_GNUC_INTERNAL
	void pcmk__unpack_location(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	pe__location_t pcmk__new_location(const char id, pe_resource_t *rsc,
	int node_weight, const char *discover_mode,
	pe_node_t *foo_node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_locations(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_location(pe__location_t constraint, pe_resource_t rsc);


	// Colocation constraints (pcmk_sched_colocation.c)

	enum pcmk__coloc_affects {
	pcmk__coloc_affects_nothing = 0,
	pcmk__coloc_affects_location,
	pcmk__coloc_affects_role,
	};

	G_GNUC_INTERNAL
	enum pcmk__coloc_affects pcmk__colocation_affects(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint,
	bool preview);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_weights(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_priority(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__apply_colocation(pcmk__colocation_t *colocation,
	pe_resource_t rsc1, pe_resource_t rsc2,
	uint32_t flags);

	G_GNUC_INTERNAL
	void pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__new_colocation(const char id, const char node_attr, int score,
	pe_resource_t dependent, pe_resource_t primary,
	const char dependent_role, const char primary_role,
	bool influence, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__block_colocated_starts(pe_action_t *action,
	pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Check whether colocation's left-hand preferences should be considered
	*
	* \param[in] colocation Colocation constraint
	* \param[in] rsc Right-hand instance (normally this will be
	* colocation->primary, which NULL will be treated as,
	* but for clones or bundles with multiple instances
	* this can be a particular instance)
	*
	* \return true if colocation influence should be effective, otherwise false
	*/
	static inline bool
	pcmk__colocation_has_influence(const pcmk__colocation_t *colocation,
	const pe_resource_t *rsc)
	{
	if (rsc == NULL) {
	rsc = colocation->primary;
	}

	/* The left hand of a colocation influences the right hand's location
	* if the influence option is true, or the right hand is not yet active.
	*/
	return colocation->influence \|\| (rsc->running_on == NULL);
	}


	// Ordering constraints (pcmk_sched_ordering.c)

	G_GNUC_INTERNAL
	void pcmk__new_ordering(pe_resource_t lh_rsc, char lh_task,
	pe_action_t lh_action, pe_resource_t rh_rsc,
	char rh_task, pe_action_t rh_action,
	enum pe_ordering type, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__disable_invalid_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_stops_before_shutdown(pe_node_t *node,
	pe_action_t *shutdown_op,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_after_each(pe_action_t after, GList list);


	/*!
	* \internal
	* \brief Create a new ordering between two resource actions
	*
	* \param[in] lh_rsc Resource for 'first' action
	* \param[in] rh_rsc Resource for 'then' action
	* \param[in] lh_task Action key for 'first' action
	* \param[in] rh_task Action key for 'then' action
	* \param[in] flags Bitmask of enum pe_ordering flags
	* \param[in] data_set Cluster working set to add ordering to
	*/
	#define pcmk__order_resource_actions(lh_rsc, lh_task, rh_rsc, rh_task, \
	flags, data_set) \
	pcmk__new_ordering((lh_rsc), pcmk__op_key((lh_rsc)->id, (lh_task), 0), \
	NULL, \
	(rh_rsc), pcmk__op_key((rh_rsc)->id, (rh_task), 0), \
	NULL, (flags), (data_set))

	#define pcmk__order_starts(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_START, \
	(rsc2), CRMD_ACTION_START, (type), (data_set))

	#define pcmk__order_stops(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_STOP, \
	(rsc2), CRMD_ACTION_STOP, (type), (data_set))

	+
	+// Ticket constraints (pcmk_sched_tickets.c)
	+
	G_GNUC_INTERNAL
	void pcmk__unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set);

	+
	+// Promotable clone resources (pcmk_sched_promotable.c)
	+
	G_GNUC_INTERNAL
	void pcmk__require_promotion_tickets(pe_resource_t *rsc);

	+G_GNUC_INTERNAL
	+void pcmk__set_instance_roles(pe_resource_t *rsc);
	+
	+
	+// Pacemaker Remote nodes (pcmk_sched_remote.c)
	+
	G_GNUC_INTERNAL
	bool pcmk__is_failed_remote_node(pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_remote_connection_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_corresponds_to_guest(pe_resource_t rsc, pe_node_t node);

	G_GNUC_INTERNAL
	pe_node_t pcmk__connection_host_for_action(pe_action_t action);

	G_GNUC_INTERNAL
	void pcmk__substitute_remote_addr(pe_resource_t rsc, GHashTable params);

	G_GNUC_INTERNAL
	void pcmk__add_bundle_meta_to_xml(xmlNode args_xml, pe_action_t action);


	// Groups (pcmk_sched_group.c)

	G_GNUC_INTERNAL
	GList pcmk__group_colocated_resources(pe_resource_t rsc,
	pe_resource_t *orig_rsc,
	GList *colocated_rscs);


	// Bundles (pcmk_sched_bundle.c)

	G_GNUC_INTERNAL
	void pcmk__output_bundle_actions(pe_resource_t *rsc);


	// Injections (pcmk_injections.c)

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node(cib_t cib_conn, const char node, const char uuid);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_node_state_change(cib_t cib_conn, const char *node,
	bool up);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_resource_history(pcmk__output_t out, xmlNode *cib_node,
	const char *resource,
	const char *lrm_name,
	const char *rclass,
	const char *rtype,
	const char *rprovider);

	G_GNUC_INTERNAL
	void pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc);

	G_GNUC_INTERNAL
	xmlNode pcmk__inject_action_result(xmlNode cib_resource,
	lrmd_event_data_t *op, int target_rc);


	// Nodes (pcmk_sched_nodes.c)

	G_GNUC_INTERNAL
	bool pcmk__node_available(const pe_node_t *node, bool consider_score,
	bool consider_guest);

	G_GNUC_INTERNAL
	bool pcmk__any_node_available(GHashTable *nodes);

	G_GNUC_INTERNAL
	GHashTable pcmk__copy_node_table(GHashTable nodes);

	G_GNUC_INTERNAL
	GList pcmk__sort_nodes(GList nodes, pe_node_t *active_node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_node_health(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	pe_node_t pcmk__top_allowed_node(const pe_resource_t rsc,
	const pe_node_t *node);


	// Clone notifictions (pcmk_sched_notif.c)

	G_GNUC_INTERNAL
	void pcmk__create_notifications(pe_resource_t rsc, notify_data_t n_data);

	G_GNUC_INTERNAL
	notify_data_t pcmk__clone_notif_pseudo_ops(pe_resource_t rsc,
	const char *task,
	pe_action_t *action,
	pe_action_t *complete);

	G_GNUC_INTERNAL
	void pcmk__free_notification_data(notify_data_t *n_data);

	G_GNUC_INTERNAL
	void pcmk__order_notifs_after_fencing(pe_action_t action, pe_resource_t rsc,
	pe_action_t *stonith_op);


	// Functions applying to more than one variant (pcmk_sched_resource.c)

	G_GNUC_INTERNAL
	void pcmk__set_allocation_methods(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_agent_changed(pe_resource_t rsc, pe_node_t node,
	const xmlNode *rsc_entry, bool active_on_node);

	G_GNUC_INTERNAL
	GList pcmk__rscs_matching_id(const char id, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	GList pcmk__colocated_resources(pe_resource_t rsc, pe_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__output_resource_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__assign_primitive(pe_resource_t rsc, pe_node_t chosen, bool force);

	G_GNUC_INTERNAL
	bool pcmk__assign_resource(pe_resource_t rsc, pe_node_t node, bool force);

	G_GNUC_INTERNAL
	void pcmk__unassign_resource(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__threshold_reached(pe_resource_t rsc, pe_node_t node,
	pe_resource_t **failed);

	G_GNUC_INTERNAL
	void pcmk__sort_resources(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance(gconstpointer a, gconstpointer b);

	G_GNUC_INTERNAL
	gint pcmk__cmp_instance_number(gconstpointer a, gconstpointer b);


	// Functions related to probes (pcmk_sched_probes.c)

	G_GNUC_INTERNAL
	void pcmk__order_probes(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__schedule_probes(pe_working_set_t *data_set);


	// Functions related to node utilization (pcmk_sched_utilization.c)

	G_GNUC_INTERNAL
	int pcmk__compare_node_capacities(const pe_node_t *node1,
	const pe_node_t *node2);

	G_GNUC_INTERNAL
	void pcmk__consume_node_capacity(GHashTable *current_utilization,
	pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__release_node_capacity(GHashTable *current_utilization,
	pe_resource_t *rsc);

	G_GNUC_INTERNAL
	void pcmk__ban_insufficient_capacity(pe_resource_t rsc, pe_node_t *prefer);

	G_GNUC_INTERNAL
	void pcmk__create_utilization_constraints(pe_resource_t *rsc,
	GList *allowed_nodes);

	G_GNUC_INTERNAL
	void pcmk__show_node_capacities(const char desc, pe_working_set_t data_set);

	#endif // PCMK__LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_sched_clone.c b/lib/pacemaker/pcmk_sched_clone.c
	index 6a4c98840c..b94a6d2ab0 100644
	--- a/lib/pacemaker/pcmk_sched_clone.c
	+++ b/lib/pacemaker/pcmk_sched_clone.c
	@@ -1,1165 +1,1168 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	static void append_parent_colocation(pe_resource_t * rsc, pe_resource_t * child, gboolean all);

	static pe_node_t *
	can_run_instance(pe_resource_t * rsc, pe_node_t * node, int limit)
	{
	pe_node_t *local_node = NULL;

	if (node == NULL && rsc->allowed_nodes) {
	GHashTableIter iter;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&local_node)) {
	can_run_instance(rsc, local_node, limit);
	}
	return NULL;
	}

	if (!node) {
	/* make clang analyzer happy */
	goto bail;

	} else if (!pcmk__node_available(node, false, false)) {
	goto bail;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	goto bail;
	}

	local_node = pcmk__top_allowed_node(rsc, node);

	if (local_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed", rsc->id, node->details->uname);
	goto bail;

	} else if (local_node->weight < 0) {
	common_update_score(rsc, node->details->id, local_node->weight);
	pe_rsc_trace(rsc, "%s cannot run on %s: Parent node weight doesn't allow it.",
	rsc->id, node->details->uname);

	} else if (local_node->count < limit) {
	pe_rsc_trace(rsc, "%s can run on %s (already running %d)",
	rsc->id, node->details->uname, local_node->count);
	return local_node;

	} else {
	pe_rsc_trace(rsc, "%s cannot run on %s: node full (%d >= %d)",
	rsc->id, node->details->uname, local_node->count, limit);
	}

	bail:
	if (node) {
	common_update_score(rsc, node->details->id, -INFINITY);
	}
	return NULL;
	}

	static pe_node_t *
	allocate_instance(pe_resource_t rsc, pe_node_t prefer, gboolean all_coloc,
	int limit, pe_working_set_t *data_set)
	{
	pe_node_t *chosen = NULL;
	GHashTable *backup = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Checking allocation of %s (preferring %s, using %s parent colocations)",
	rsc->id, (prefer? prefer->details->uname: "none"),
	(all_coloc? "all" : "some"));

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->fns->location(rsc, NULL, FALSE);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	/* Only include positive colocation preferences of dependent resources
	* if not every node will get a copy of the clone
	*/
	append_parent_colocation(rsc->parent, rsc, all_coloc);

	if (prefer) {
	pe_node_t *local_prefer = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (local_prefer == NULL \|\| local_prefer->weight < 0) {
	pe_rsc_trace(rsc, "Not pre-allocating %s to %s - unavailable", rsc->id,
	prefer->details->uname);
	return NULL;
	}
	}

	can_run_instance(rsc, NULL, limit);

	backup = pcmk__copy_node_table(rsc->allowed_nodes);
	pe_rsc_trace(rsc, "Allocating instance %s", rsc->id);
	chosen = rsc->cmds->allocate(rsc, prefer, data_set);
	if (chosen && prefer && (chosen->details != prefer->details)) {
	crm_info("Not pre-allocating %s to %s because %s is better",
	rsc->id, prefer->details->uname, chosen->details->uname);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = backup;
	pcmk__unassign_resource(rsc);
	chosen = NULL;
	backup = NULL;
	}
	if (chosen) {
	pe_node_t *local_node = pcmk__top_allowed_node(rsc, chosen);

	if (local_node) {
	local_node->count++;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	/* what to do? we can't enforce per-node limits in this case */
	pcmk__config_err("%s not found in %s (list of %d)",
	chosen->details->id, rsc->parent->id,
	g_hash_table_size(rsc->parent->allowed_nodes));
	}
	}

	if(backup) {
	g_hash_table_destroy(backup);
	}
	return chosen;
	}

	static void
	append_parent_colocation(pe_resource_t * rsc, pe_resource_t * child, gboolean all)
	{

	GList *gIter = NULL;

	gIter = rsc->rsc_cons;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) gIter->data;

	if (all \|\| cons->score < 0 \|\| cons->score == INFINITY) {
	child->rsc_cons = g_list_prepend(child->rsc_cons, cons);
	}
	}

	gIter = rsc->rsc_cons_lhs;
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) gIter->data;

	if (!pcmk__colocation_has_influence(cons, child)) {
	continue;
	}
	if (all \|\| cons->score < 0) {
	child->rsc_cons_lhs = g_list_prepend(child->rsc_cons_lhs, cons);
	}
	}
	}


	void
	distribute_children(pe_resource_t rsc, GList children, GList *nodes,
	int max, int per_host_max, pe_working_set_t * data_set);

	void
	distribute_children(pe_resource_t rsc, GList children, GList *nodes,
	int max, int per_host_max, pe_working_set_t * data_set)
	{
	int loop_max = 0;
	int allocated = 0;
	int available_nodes = 0;
	bool all_coloc = false;

	/* count now tracks the number of clones currently allocated */
	for(GList *nIter = nodes; nIter != NULL; nIter = nIter->next) {
	pe_node_t *node = nIter->data;

	node->count = 0;
	if (pcmk__node_available(node, false, false)) {
	available_nodes++;
	}
	}

	all_coloc = (max < available_nodes) ? true : false;

	if(available_nodes) {
	loop_max = max / available_nodes;
	}
	if (loop_max < 1) {
	loop_max = 1;
	}

	pe_rsc_debug(rsc, "Allocating up to %d %s instances to a possible %d nodes (at most %d per host, %d optimal)",
	max, rsc->id, available_nodes, per_host_max, loop_max);

	/* Pre-allocate as many instances as we can to their current location */
	for (GList *gIter = children; gIter != NULL && allocated < max; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;
	pe_node_t *child_node = NULL;
	pe_node_t *local_node = NULL;

	if ((child->running_on == NULL)
	\|\| !pcmk_is_set(child->flags, pe_rsc_provisional)
	\|\| pcmk_is_set(child->flags, pe_rsc_failed)) {

	continue;
	}

	child_node = pe__current_node(child);
	local_node = pcmk__top_allowed_node(child, child_node);

	pe_rsc_trace(rsc,
	"Checking pre-allocation of %s to %s (%d remaining of %d)",
	child->id, child_node->details->uname, max - allocated,
	max);

	if (!pcmk__node_available(child_node, true, false)) {
	pe_rsc_trace(rsc, "Not pre-allocating because %s can not run %s",
	child_node->details->uname, child->id);
	continue;
	}

	if ((local_node != NULL) && (local_node->count >= loop_max)) {
	pe_rsc_trace(rsc,
	"Not pre-allocating because %s already allocated "
	"optimal instances", child_node->details->uname);
	continue;
	}

	if (allocate_instance(child, child_node, all_coloc, per_host_max,
	data_set)) {
	pe_rsc_trace(rsc, "Pre-allocated %s to %s", child->id,
	child_node->details->uname);
	allocated++;
	}
	}

	pe_rsc_trace(rsc, "Done pre-allocating (%d of %d)", allocated, max);

	for (GList *gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	if (child->running_on != NULL) {
	pe_node_t *child_node = pe__current_node(child);
	pe_node_t *local_node = pcmk__top_allowed_node(child, child_node);

	if (local_node == NULL) {
	crm_err("%s is running on %s which isn't allowed",
	child->id, child_node->details->uname);
	}
	}

	if (!pcmk_is_set(child->flags, pe_rsc_provisional)) {
	} else if (allocated >= max) {
	pe_rsc_debug(rsc, "Child %s not allocated - limit reached %d %d", child->id, allocated, max);
	resource_location(child, NULL, -INFINITY, "clone:limit_reached", data_set);
	} else {
	if (allocate_instance(child, NULL, all_coloc, per_host_max,
	data_set)) {
	allocated++;
	}
	}
	}

	pe_rsc_debug(rsc, "Allocated %d %s instances of a possible %d",
	allocated, rsc->id, max);
	}


	pe_node_t *
	pcmk__clone_allocate(pe_resource_t rsc, pe_node_t prefer,
	pe_working_set_t *data_set)
	{
	GList *nodes = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pcmk__add_promotion_scores(rsc);
	}

	pe__set_resource_flags(rsc, pe_rsc_allocating);

	/* This information is used by pcmk__cmp_instance() when deciding the order
	* in which to assign clone instances to nodes.
	*/
	for (GList *gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	pe_rsc_trace(rsc, "%s: Allocating %s first",
	rsc->id, constraint->primary->id);
	constraint->primary->cmds->allocate(constraint->primary, prefer,
	data_set);
	}

	for (GList *gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	pcmk__colocation_t constraint = (pcmk__colocation_t ) gIter->data;

	if (pcmk__colocation_has_influence(constraint, NULL)) {
	pcmk__apply_colocation(constraint, rsc, constraint->dependent,
	pe_weights_rollback\|pe_weights_positive);
	}
	}

	pe__show_node_weights(!pcmk_is_set(data_set->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, data_set);

	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = pcmk__sort_nodes(nodes, NULL, data_set);
	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance);
	distribute_children(rsc, rsc->children, nodes, clone_data->clone_max, clone_data->clone_node_max, data_set);
	g_list_free(nodes);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	- pcmk__set_instance_roles(rsc, data_set);
	+ pcmk__set_instance_roles(rsc);
	}

	pe__clear_resource_flags(rsc, pe_rsc_provisional\|pe_rsc_allocating);
	pe_rsc_trace(rsc, "Done allocating %s", rsc->id);
	return NULL;
	}

	static void
	clone_update_pseudo_status(pe_resource_t * rsc, gboolean * stopping, gboolean * starting,
	gboolean * active)
	{
	GList *gIter = NULL;

	if (rsc->children) {

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	clone_update_pseudo_status(child, stopping, starting, active);
	}

	return;
	}

	CRM_ASSERT(active != NULL);
	CRM_ASSERT(starting != NULL);
	CRM_ASSERT(stopping != NULL);

	if (rsc->running_on) {
	*active = TRUE;
	}

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (starting && stopping) {
	return;

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "Skipping optional: %s", action->uuid);
	continue;

	} else if (!pcmk_any_flags_set(action->flags,
	pe_action_pseudo\|pe_action_runnable)) {
	pe_rsc_trace(rsc, "Skipping unrunnable: %s", action->uuid);
	continue;

	} else if (pcmk__str_eq(RSC_STOP, action->task, pcmk__str_casei)) {
	pe_rsc_trace(rsc, "Stopping due to: %s", action->uuid);
	*stopping = TRUE;

	} else if (pcmk__str_eq(RSC_START, action->task, pcmk__str_casei)) {
	if (!pcmk_is_set(action->flags, pe_action_runnable)) {
	pe_rsc_trace(rsc, "Skipping pseudo-op: %s run=%d, pseudo=%d",
	action->uuid,
	pcmk_is_set(action->flags, pe_action_runnable),
	pcmk_is_set(action->flags, pe_action_pseudo));
	} else {
	pe_rsc_trace(rsc, "Starting due to: %s", action->uuid);
	pe_rsc_trace(rsc, "%s run=%d, pseudo=%d",
	action->uuid,
	pcmk_is_set(action->flags, pe_action_runnable),
	pcmk_is_set(action->flags, pe_action_pseudo));
	*starting = TRUE;
	}
	}
	}
	}

	static pe_action_t *
	find_rsc_action(pe_resource_t rsc, const char task)
	{
	pe_action_t *match = NULL;
	GList *actions = pe__resource_actions(rsc, NULL, task, FALSE);

	for (GList *item = actions; item != NULL; item = item->next) {
	pe_action_t op = (pe_action_t ) item->data;

	if (!pcmk_is_set(op->flags, pe_action_optional)) {
	if (match != NULL) {
	// More than one match, don't return any
	match = NULL;
	break;
	}
	match = op;
	}
	}
	g_list_free(actions);
	return match;
	}

	static void
	child_ordering_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *stop = NULL;
	pe_action_t *start = NULL;
	pe_action_t *last_stop = NULL;
	pe_action_t *last_start = NULL;
	GList *gIter = NULL;

	if (!pe__clone_is_ordered(rsc)) {
	return;
	}

	/* we have to maintain a consistent sorted child list when building order constraints */
	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance_number);

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	stop = find_rsc_action(child, RSC_STOP);
	if (stop) {
	if (last_stop) {
	/* child/child relative stop */
	order_actions(stop, last_stop, pe_order_optional);
	}
	last_stop = stop;
	}

	start = find_rsc_action(child, RSC_START);
	if (start) {
	if (last_start) {
	/* child/child relative start */
	order_actions(last_start, start, pe_order_optional);
	}
	last_start = start;
	}
	}
	}

	void
	clone_create_actions(pe_resource_t rsc, pe_working_set_t data_set)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);
	clone_create_pseudo_actions(rsc, rsc->children, &clone_data->start_notify, &clone_data->stop_notify,data_set);
	child_ordering_constraints(rsc, data_set);
	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	create_promotable_actions(rsc, data_set);
	}
	}

	void
	clone_create_pseudo_actions(
	pe_resource_t * rsc, GList children, notify_data_t start_notify, notify_data_t stop_notify, pe_working_set_t data_set)
	{
	gboolean child_active = FALSE;
	gboolean child_starting = FALSE;
	gboolean child_stopping = FALSE;
	gboolean allow_dependent_migrations = TRUE;

	pe_action_t *stop = NULL;
	pe_action_t *stopped = NULL;

	pe_action_t *start = NULL;
	pe_action_t *started = NULL;

	pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);

	for (GList *gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean starting = FALSE;
	gboolean stopping = FALSE;

	child_rsc->cmds->create_actions(child_rsc, data_set);
	clone_update_pseudo_status(child_rsc, &stopping, &starting, &child_active);
	if (stopping && starting) {
	allow_dependent_migrations = FALSE;
	}

	child_stopping \|= stopping;
	child_starting \|= starting;
	}

	/* start */
	start = pcmk__new_rsc_pseudo_action(rsc, RSC_START, !child_starting, true);
	started = pcmk__new_rsc_pseudo_action(rsc, RSC_STARTED, !child_starting,
	false);
	started->priority = INFINITY;

	if (child_active \|\| child_starting) {
	pe__set_action_flags(started, pe_action_runnable);
	}

	if (start_notify != NULL && *start_notify == NULL) {
	*start_notify = pcmk__clone_notif_pseudo_ops(rsc, RSC_START, start,
	started);
	}

	/* stop */
	stop = pcmk__new_rsc_pseudo_action(rsc, RSC_STOP, !child_stopping, true);
	stopped = pcmk__new_rsc_pseudo_action(rsc, RSC_STOPPED, !child_stopping,
	true);
	stopped->priority = INFINITY;
	if (allow_dependent_migrations) {
	pe__set_action_flags(stop, pe_action_migrate_runnable);
	}

	if (stop_notify != NULL && *stop_notify == NULL) {
	*stop_notify = pcmk__clone_notif_pseudo_ops(rsc, RSC_STOP, stop,
	stopped);

	if (start_notify && start_notify && stop_notify) {
	order_actions((stop_notify)->post_done, (start_notify)->pre, pe_order_optional);
	}
	}
	}

	void
	clone_internal_constraints(pe_resource_t rsc, pe_working_set_t data_set)
	{
	pe_resource_t *last_rsc = NULL;
	GList *gIter;
	bool ordered = pe__clone_is_ordered(rsc);

	pe_rsc_trace(rsc, "Internal constraints for %s", rsc->id);
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_START,
	pe_order_optional, data_set);
	pcmk__order_resource_actions(rsc, RSC_START, rsc, RSC_STARTED,
	pe_order_runnable_left, data_set);
	pcmk__order_resource_actions(rsc, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_runnable_left, data_set);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_STOP,
	pe_order_optional, data_set);
	pcmk__order_resource_actions(rsc, RSC_STARTED, rsc, RSC_PROMOTE,
	pe_order_runnable_left, data_set);
	}

	if (ordered) {
	/* we have to maintain a consistent sorted child list when building order constraints */
	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance_number);
	}
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	pcmk__order_starts(rsc, child_rsc,
	pe_order_runnable_left\|pe_order_implies_first_printed,
	data_set);
	pcmk__order_resource_actions(child_rsc, RSC_START, rsc, RSC_STARTED,
	pe_order_implies_then_printed, data_set);
	if (ordered && (last_rsc != NULL)) {
	pcmk__order_starts(last_rsc, child_rsc, pe_order_optional,
	data_set);
	}

	pcmk__order_stops(rsc, child_rsc, pe_order_implies_first_printed,
	data_set);
	pcmk__order_resource_actions(child_rsc, RSC_STOP, rsc, RSC_STOPPED,
	pe_order_implies_then_printed, data_set);
	if (ordered && (last_rsc != NULL)) {
	pcmk__order_stops(child_rsc, last_rsc, pe_order_optional, data_set);
	}

	last_rsc = child_rsc;
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	promotable_constraints(rsc, data_set);
	}
	}

	gboolean
	is_child_compatible(pe_resource_t child_rsc, pe_node_t local_node, enum rsc_role_e filter, gboolean current)
	{
	pe_node_t *node = NULL;
	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, current);

	CRM_CHECK(child_rsc && local_node, return FALSE);
	if (is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
	/* We only want instances that haven't failed */
	node = child_rsc->fns->location(child_rsc, NULL, current);
	}

	if (filter != RSC_ROLE_UNKNOWN && next_role != filter) {
	crm_trace("Filtered %s", child_rsc->id);
	return FALSE;
	}

	if (node && (node->details == local_node->details)) {
	return TRUE;

	} else if (node) {
	crm_trace("%s - %s vs %s", child_rsc->id, node->details->uname,
	local_node->details->uname);

	} else {
	crm_trace("%s - not allocated %d", child_rsc->id, current);
	}
	return FALSE;
	}

	pe_resource_t *
	find_compatible_child(pe_resource_t local_child, pe_resource_t rsc,
	enum rsc_role_e filter, gboolean current,
	pe_working_set_t *data_set)
	{
	pe_resource_t *pair = NULL;
	GList *gIter = NULL;
	GList *scratch = NULL;
	pe_node_t *local_node = NULL;

	local_node = local_child->fns->location(local_child, NULL, current);
	if (local_node) {
	return find_compatible_child_by_node(local_child, local_node, rsc, filter, current);
	}

	scratch = g_hash_table_get_values(local_child->allowed_nodes);
	scratch = pcmk__sort_nodes(scratch, NULL, data_set);

	gIter = scratch;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	pair = find_compatible_child_by_node(local_child, node, rsc, filter, current);
	if (pair) {
	goto done;
	}
	}

	pe_rsc_debug(rsc, "Can't pair %s with %s", local_child->id, rsc->id);
	done:
	g_list_free(scratch);
	return pair;
	}

	void
	clone_rsc_colocation_lh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	/* -- Never called --
	*
	* Instead we add the colocation constraints to the child and call from there
	*/
	CRM_ASSERT(FALSE);
	}

	void
	clone_rsc_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;
	gboolean do_interleave = FALSE;
	const char *interleave_s = NULL;

	CRM_CHECK(constraint != NULL, return);
	CRM_CHECK(dependent != NULL,
	pe_err("dependent was NULL for %s", constraint->id); return);
	CRM_CHECK(primary != NULL,
	pe_err("primary was NULL for %s", constraint->id); return);
	CRM_CHECK(dependent->variant == pe_native, return);

	pe_rsc_trace(primary, "Processing constraint %s: %s -> %s %d",
	constraint->id, dependent->id, primary->id, constraint->score);

	if (pcmk_is_set(primary->flags, pe_rsc_promotable)) {
	if (pcmk_is_set(primary->flags, pe_rsc_provisional)) {
	pe_rsc_trace(primary, "%s is still provisional", primary->id);
	return;
	} else if (constraint->primary_role == RSC_ROLE_UNKNOWN) {
	pe_rsc_trace(primary, "Handling %s as a clone colocation",
	constraint->id);
	} else {
	promotable_colocation_rh(dependent, primary, constraint, data_set);
	return;
	}
	}

	/* only the LHS side needs to be labeled as interleave */
	interleave_s = g_hash_table_lookup(constraint->dependent->meta,
	XML_RSC_ATTR_INTERLEAVE);
	if (crm_is_true(interleave_s)
	&& (constraint->dependent->variant > pe_group)) {
	/* @TODO Do we actually care about multiple primary copies sharing a
	* dependent copy anymore?
	*/
	if (copies_per_node(constraint->dependent) != copies_per_node(constraint->primary)) {
	pcmk__config_err("Cannot interleave %s and %s because they do not "
	"support the same number of instances per node",
	constraint->dependent->id,
	constraint->primary->id);

	} else {
	do_interleave = TRUE;
	}
	}

	if (pcmk_is_set(primary->flags, pe_rsc_provisional)) {
	pe_rsc_trace(primary, "%s is still provisional", primary->id);
	return;

	} else if (do_interleave) {
	pe_resource_t *primary_instance = NULL;

	primary_instance = find_compatible_child(dependent, primary,
	RSC_ROLE_UNKNOWN, FALSE,
	data_set);
	if (primary_instance != NULL) {
	pe_rsc_debug(primary, "Pairing %s with %s",
	dependent->id, primary_instance->id);
	dependent->cmds->rsc_colocation_lh(dependent, primary_instance,
	constraint, data_set);

	} else if (constraint->score >= INFINITY) {
	crm_notice("Cannot pair %s with instance of %s",
	dependent->id, primary->id);
	pcmk__assign_resource(dependent, NULL, true);

	} else {
	pe_rsc_debug(primary, "Cannot pair %s with instance of %s",
	dependent->id, primary->id);
	}

	return;

	} else if (constraint->score >= INFINITY) {
	GList *affected_nodes = NULL;

	gIter = primary->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	pe_node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);

	if (chosen != NULL && is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
	pe_rsc_trace(primary, "Allowing %s: %s %d",
	constraint->id, chosen->details->uname,
	chosen->weight);
	affected_nodes = g_list_prepend(affected_nodes, chosen);
	}
	}

	node_list_exclude(dependent->allowed_nodes, affected_nodes, FALSE);
	g_list_free(affected_nodes);
	return;
	}

	gIter = primary->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_rh(dependent, child_rsc, constraint,
	data_set);
	}
	}

	enum action_tasks
	clone_child_action(pe_action_t * action)
	{
	enum action_tasks result = no_action;
	pe_resource_t child = (pe_resource_t ) action->rsc->children->data;

	if (pcmk__strcase_any_of(action->task, "notify", "notified", NULL)) {

	/* Find the action we're notifying about instead */

	int stop = 0;
	char *key = action->uuid;
	int lpc = strlen(key);

	for (; lpc > 0; lpc--) {
	if (key[lpc] == '_' && stop == 0) {
	stop = lpc;

	} else if (key[lpc] == '_') {
	char *task_mutable = NULL;

	lpc++;
	task_mutable = strdup(key + lpc);
	task_mutable[stop - lpc] = 0;

	crm_trace("Extracted action '%s' from '%s'", task_mutable, key);
	result = get_complex_task(child, task_mutable, TRUE);
	free(task_mutable);
	break;
	}
	}

	} else {
	result = get_complex_task(child, action->task, TRUE);
	}
	return result;
	}

	#define pe__clear_action_summary_flags(flags, action, flag) do { \
	flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Action summary", action->rsc->id, \
	flags, flag, #flag); \
	} while (0)

	enum pe_action_flags
	summary_action_flags(pe_action_t * action, GList children, pe_node_t node)
	{
	GList *gIter = NULL;
	gboolean any_runnable = FALSE;
	gboolean check_runnable = TRUE;
	enum action_tasks task = clone_child_action(action);
	enum pe_action_flags flags = (pe_action_optional \| pe_action_runnable \| pe_action_pseudo);
	const char *task_s = task2text(task);

	for (gIter = children; gIter != NULL; gIter = gIter->next) {
	pe_action_t *child_action = NULL;
	pe_resource_t child = (pe_resource_t ) gIter->data;

	child_action = find_first_action(child->actions, NULL, task_s, child->children ? NULL : node);
	pe_rsc_trace(action->rsc, "Checking for %s in %s on %s (%s)", task_s, child->id,
	node ? node->details->uname : "none", child_action?child_action->uuid:"NA");
	if (child_action) {
	enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);

	if (pcmk_is_set(flags, pe_action_optional)
	&& !pcmk_is_set(child_flags, pe_action_optional)) {
	pe_rsc_trace(child, "%s is mandatory because of %s", action->uuid,
	child_action->uuid);
	pe__clear_action_summary_flags(flags, action, pe_action_optional);
	pe__clear_action_flags(action, pe_action_optional);
	}
	if (pcmk_is_set(child_flags, pe_action_runnable)) {
	any_runnable = TRUE;
	}
	}
	}

	if (check_runnable && any_runnable == FALSE) {
	pe_rsc_trace(action->rsc, "%s is not runnable because no children are", action->uuid);
	pe__clear_action_summary_flags(flags, action, pe_action_runnable);
	if (node == NULL) {
	pe__clear_action_flags(action, pe_action_runnable);
	}
	}

	return flags;
	}

	enum pe_action_flags
	clone_action_flags(pe_action_t * action, pe_node_t * node)
	{
	return summary_action_flags(action, action->rsc->children, node);
	}

	void
	clone_rsc_location(pe_resource_t rsc, pe__location_t constraint)
	{
	GList *gIter = rsc->children;

	pe_rsc_trace(rsc, "Processing location constraint %s for %s", constraint->id, rsc->id);

	pcmk__apply_location(constraint, rsc);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->rsc_location(child_rsc, constraint);
	}
	}

	void
	clone_expand(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	g_list_foreach(rsc->actions, (GFunc) rsc->cmds->action_flags, NULL);

	pcmk__create_notifications(rsc, clone_data->start_notify);
	pcmk__create_notifications(rsc, clone_data->stop_notify);
	pcmk__create_notifications(rsc, clone_data->promote_notify);
	pcmk__create_notifications(rsc, clone_data->demote_notify);

	/* Now that the notifcations have been created we can expand the children */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}

	native_expand(rsc, data_set);

	/* The notifications are in the graph now, we can destroy the notify_data */
	pcmk__free_notification_data(clone_data->demote_notify);
	clone_data->demote_notify = NULL;
	pcmk__free_notification_data(clone_data->stop_notify);
	clone_data->stop_notify = NULL;
	pcmk__free_notification_data(clone_data->start_notify);
	clone_data->start_notify = NULL;
	pcmk__free_notification_data(clone_data->promote_notify);
	clone_data->promote_notify = NULL;
	}

	// Check whether a resource or any of its children is known on node
	static bool
	rsc_known_on(const pe_resource_t rsc, const pe_node_t node)
	{
	if (rsc->children) {
	for (GList *child_iter = rsc->children; child_iter != NULL;
	child_iter = child_iter->next) {

	pe_resource_t child = (pe_resource_t ) child_iter->data;

	if (rsc_known_on(child, node)) {
	return TRUE;
	}
	}

	} else if (rsc->known_on) {
	GHashTableIter iter;
	pe_node_t *known_node = NULL;

	g_hash_table_iter_init(&iter, rsc->known_on);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &known_node)) {
	if (node->details == known_node->details) {
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Look for an instance of clone that is known on node
	static pe_resource_t *
	find_instance_on(const pe_resource_t clone, const pe_node_t node)
	{
	for (GList *gIter = clone->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	if (rsc_known_on(child, node)) {
	return child;
	}
	}
	return NULL;
	}

	// For unique clones, probe each instance separately
	static gboolean
	probe_unique_clone(pe_resource_t rsc, pe_node_t node, pe_action_t *complete,
	gboolean force, pe_working_set_t *data_set)
	{
	gboolean any_created = FALSE;

	for (GList *child_iter = rsc->children; child_iter != NULL;
	child_iter = child_iter->next) {

	pe_resource_t child = (pe_resource_t ) child_iter->data;

	any_created \|= child->cmds->create_probe(child, node, complete, force,
	data_set);
	}
	return any_created;
	}

	// For anonymous clones, only a single instance needs to be probed
	static gboolean
	probe_anonymous_clone(pe_resource_t rsc, pe_node_t node,
	pe_action_t *complete, gboolean force,
	pe_working_set_t *data_set)
	{
	// First, check if we probed an instance on this node last time
	pe_resource_t *child = find_instance_on(rsc, node);

	// Otherwise, check if we plan to start an instance on this node
	if (child == NULL) {
	for (GList *child_iter = rsc->children; child_iter && !child;
	child_iter = child_iter->next) {

	pe_node_t *local_node = NULL;
	pe_resource_t child_rsc = (pe_resource_t ) child_iter->data;

	if (child_rsc) { /* make clang analyzer happy */
	local_node = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if (local_node && (local_node->details == node->details)) {
	child = child_rsc;
	}
	}
	}
	}

	// Otherwise, use the first clone instance
	if (child == NULL) {
	child = rsc->children->data;
	}
	CRM_ASSERT(child);
	return child->cmds->create_probe(child, node, complete, force, data_set);
	}

	gboolean
	clone_create_probe(pe_resource_t * rsc, pe_node_t * node, pe_action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	gboolean any_created = FALSE;

	CRM_ASSERT(rsc);

	rsc->children = g_list_sort(rsc->children, pcmk__cmp_instance_number);
	if (rsc->children == NULL) {
	pe_warn("Clone %s has no children", rsc->id);
	return FALSE;
	}

	if (rsc->exclusive_discover) {
	pe_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (allowed && allowed->rsc_discover_mode != pe_discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on
	*
	* remove the node from allowed_nodes so that the
	* notification contains only nodes that we might ever run
	* on
	*/
	g_hash_table_remove(rsc->allowed_nodes, node->details->id);

	/* Bit of a shortcut - might as well take it */
	return FALSE;
	}
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	any_created = probe_unique_clone(rsc, node, complete, force, data_set);
	} else {
	any_created = probe_anonymous_clone(rsc, node, complete, force,
	data_set);
	}
	return any_created;
	}

	void
	clone_append_meta(pe_resource_t * rsc, xmlNode * xml)
	{
	char *name = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	name = crm_meta_name(XML_RSC_ATTR_UNIQUE);
	crm_xml_add(xml, name, pe__rsc_bool_str(rsc, pe_rsc_unique));
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_NOTIFY);
	crm_xml_add(xml, name, pe__rsc_bool_str(rsc, pe_rsc_notify));
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION_MAX);
	crm_xml_add_int(xml, name, clone_data->clone_max);
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION_NODEMAX);
	crm_xml_add_int(xml, name, clone_data->clone_node_max);
	free(name);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	+ int promoted_max = pe__clone_promoted_max(rsc);
	+ int promoted_node_max = pe__clone_promoted_node_max(rsc);
	+
	name = crm_meta_name(XML_RSC_ATTR_PROMOTED_MAX);
	- crm_xml_add_int(xml, name, clone_data->promoted_max);
	+ crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_PROMOTED_NODEMAX);
	- crm_xml_add_int(xml, name, clone_data->promoted_node_max);
	+ crm_xml_add_int(xml, name, promoted_node_max);
	free(name);

	/* @COMPAT Maintain backward compatibility with resource agents that
	* expect the old names (deprecated since 2.0.0).
	*/
	name = crm_meta_name(PCMK_XE_PROMOTED_MAX_LEGACY);
	- crm_xml_add_int(xml, name, clone_data->promoted_max);
	+ crm_xml_add_int(xml, name, promoted_max);
	free(name);

	name = crm_meta_name(PCMK_XE_PROMOTED_NODE_MAX_LEGACY);
	- crm_xml_add_int(xml, name, clone_data->promoted_node_max);
	+ crm_xml_add_int(xml, name, promoted_node_max);
	free(name);
	}
	}

	// Clone implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__clone_add_utilization(pe_resource_t rsc, pe_resource_t orig_rsc,
	GList all_rscs, GHashTable utilization)
	{
	bool existing = false;
	pe_resource_t *child = NULL;

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return;
	}

	// Look for any child already existing in the list
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	child = (pe_resource_t *) iter->data;
	if (g_list_find(all_rscs, child)) {
	existing = true; // Keep checking remaining children
	} else {
	// If this is a clone of a group, look for group's members
	for (GList *member_iter = child->children; member_iter != NULL;
	member_iter = member_iter->next) {

	pe_resource_t member = (pe_resource_t ) member_iter->data;

	if (g_list_find(all_rscs, member) != NULL) {
	// Add child's utilization, not group member's
	child->cmds->add_utilization(child, orig_rsc, all_rscs,
	utilization);
	existing = true;
	break;
	}
	}
	}
	}

	if (!existing && (rsc->children != NULL)) {
	// If nothing was found, still add first child's utilization
	child = (pe_resource_t *) rsc->children->data;

	child->cmds->add_utilization(child, orig_rsc, all_rscs, utilization);
	}
	}

	// Clone implementation of resource_alloc_functions_t:shutdown_lock()
	void
	pcmk__clone_shutdown_lock(pe_resource_t *rsc)
	{
	return; // Clones currently don't support shutdown locks
	}
	diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
	index b7e08ef83a..86746dc76e 100644
	--- a/lib/pacemaker/pcmk_sched_promotable.c
	+++ b/lib/pacemaker/pcmk_sched_promotable.c
	@@ -1,1182 +1,1252 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	/*!
	* \internal
	* \brief Add implicit promotion ordering for a promotable instance
	*
	* \param[in] clone Clone resource
	* \param[in] child Instance of \p clone being ordered
	* \param[in] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_promotion(pe_resource_t clone, pe_resource_t child,
	pe_resource_t *last)
	{
	// "Promote clone" -> promote instance -> "clone promoted"
	pcmk__order_resource_actions(clone, RSC_PROMOTE, child, RSC_PROMOTE,
	pe_order_optional, clone->cluster);
	pcmk__order_resource_actions(child, RSC_PROMOTE, clone, RSC_PROMOTED,
	pe_order_optional, clone->cluster);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(last, RSC_PROMOTE, child, RSC_PROMOTE,
	pe_order_optional, clone->cluster);
	}
	}

	/*!
	* \internal
	* \brief Add implicit demotion ordering for a promotable instance
	*
	* \param[in] clone Clone resource
	* \param[in] child Instance of \p clone being ordered
	* \param[in] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_demotion(pe_resource_t clone, pe_resource_t child,
	pe_resource_t *last)
	{
	// "Demote clone" -> demote instance -> "clone demoted"
	pcmk__order_resource_actions(clone, RSC_DEMOTE, child, RSC_DEMOTE,
	pe_order_implies_first_printed,
	clone->cluster);
	pcmk__order_resource_actions(child, RSC_DEMOTE, clone, RSC_DEMOTED,
	pe_order_implies_then_printed, clone->cluster);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(child, RSC_DEMOTE, last, RSC_DEMOTE,
	pe_order_optional, clone->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance will be promoted or demoted
	*
	* \param[in] rsc Instance to check
	* \param[in] demoting If \p rsc will be demoted, this will be set to true
	* \param[in] promoting If \p rsc will be promoted, this will be set to true
	*/
	static void
	check_for_role_change(pe_resource_t rsc, bool demoting, bool *promoting)
	{
	GList *iter = NULL;

	// If this is a cloned group, check group members recursively
	if (rsc->children != NULL) {
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	check_for_role_change((pe_resource_t *) iter->data,
	demoting, promoting);
	}
	return;
	}

	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;

	if (promoting && demoting) {
	return;

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;

	} else if (pcmk__str_eq(RSC_DEMOTE, action->task, pcmk__str_none)) {
	*demoting = true;

	} else if (pcmk__str_eq(RSC_PROMOTE, action->task, pcmk__str_none)) {
	*promoting = true;
	}
	}
	}

	/*!
	* \internal
	* \brief Add promoted-role location constraint scores to an instance's priority
	*
	* Adjust a promotable clone instance's promotion priority by the scores of any
	* location constraints in a list that are both limited to the promoted role and
	* for the node where the instance will be placed.
	*
	* \param[in] child Promotable clone instance
	* \param[in] location_constraints List of location constraints to apply
	* \param[in] chosen Node where \p child will be placed
	*/
	static void
	apply_promoted_locations(pe_resource_t child, GList location_constraints,
	pe_node_t *chosen)
	{
	for (GList *iter = location_constraints; iter; iter = iter->next) {
	pe__location_t *location = iter->data;
	pe_node_t *weighted_node = NULL;

	if (location->role_filter == RSC_ROLE_PROMOTED) {
	weighted_node = pe_find_node_id(location->node_list_rh,
	chosen->details->id);
	}
	if (weighted_node != NULL) {
	int new_priority = pcmk__add_scores(child->priority,
	weighted_node->weight);

	pe_rsc_trace(child,
	"Applying location %s to %s promotion priority on %s: "
	"%d + %d = %d",
	location->id, child->id, weighted_node->details->uname,
	child->priority, weighted_node->weight, new_priority);
	child->priority = new_priority;
	}
	}
	}

	/*!
	* \internal
	* \brief Get the node that an instance will be promoted on
	*
	* \param[in] rsc Promotable clone instance to check
	*
	* \return Node that \p rsc will be promoted on, or NULL if none
	*/
	static pe_node_t *
	node_to_be_promoted_on(pe_resource_t *rsc)
	{
	pe_node_t *node = NULL;
	pe_node_t *local_node = NULL;
	pe_resource_t *parent = uber_parent(rsc);
	- clone_variant_data_t *clone_data = NULL;

	// If this is a cloned group, bail if any group member can't be promoted
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	if (node_to_be_promoted_on(child) == NULL) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because member %s can't",
	rsc->id, child->id);
	return NULL;
	}
	}

	node = rsc->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pe_rsc_trace(rsc, "%s can't be promoted because it won't be active",
	rsc->id);
	return NULL;

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	if (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED) {
	crm_notice("Unmanaged instance %s will be left promoted on %s",
	rsc->id, node->details->uname);
	} else {
	pe_rsc_trace(rsc, "%s can't be promoted because it is unmanaged",
	rsc->id);
	return NULL;
	}

	} else if (rsc->priority < 0) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because its promotion priority %d "
	"is negative",
	rsc->id, rsc->priority);
	return NULL;

	} else if (!pcmk__node_available(node, false, true)) {
	pe_rsc_trace(rsc, "%s can't be promoted because %s can't run resources",
	rsc->id, node->details->uname);
	return NULL;
	}

	- get_clone_variant_data(clone_data, parent);
	local_node = pe_hash_table_lookup(parent->allowed_nodes, node->details->id);

	if (local_node == NULL) {
	/* It should not be possible for the scheduler to have allocated the
	* instance to a node where its parent is not allowed, but it's good to
	* have a fail-safe.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	crm_warn("%s can't be promoted because %s is not allowed on %s "
	"(scheduler bug?)",
	rsc->id, parent->id, node->details->uname);
	} // else the instance is unmanaged and already promoted
	return NULL;

	- } else if ((local_node->count >= clone_data->promoted_node_max)
	+ } else if ((local_node->count >= pe__clone_promoted_node_max(parent))
	&& pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because %s has "
	"maximum promoted instances already",
	rsc->id, node->details->uname);
	return NULL;
	}

	return local_node;
	}

	/*!
	* \internal
	* \brief Compare two promotable clone instances by promotion priority
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a has higher promotion priority,
	* a positive number if \p b has higher promotion priority,
	* or 0 if promotion priorities are equal
	*/
	static gint
	cmp_promotable_instance(gconstpointer a, gconstpointer b)
	{
	const pe_resource_t rsc1 = (const pe_resource_t ) a;
	const pe_resource_t rsc2 = (const pe_resource_t ) b;

	enum rsc_role_e role1 = RSC_ROLE_UNKNOWN;
	enum rsc_role_e role2 = RSC_ROLE_UNKNOWN;

	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	// Check sort index set by pcmk__set_instance_roles()
	if (rsc1->sort_index > rsc2->sort_index) {
	pe_rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(sort index %d > %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return -1;
	} else if (rsc1->sort_index < rsc2->sort_index) {
	pe_rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(sort index %d < %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return 1;
	}

	// If those are the same, prefer instance whose current role is higher
	role1 = rsc1->fns->state(rsc1, TRUE);
	role2 = rsc2->fns->state(rsc2, TRUE);
	if (role1 > role2) {
	pe_rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(higher current role)",
	rsc1->id, rsc2->id);
	return -1;
	} else if (role1 < role2) {
	pe_rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(lower current role)",
	rsc1->id, rsc2->id);
	return 1;
	}

	// Finally, do normal clone instance sorting
	return pcmk__cmp_instance(a, b);
	}

	/*!
	* \internal
	* \brief Add a promotable clone instance's sort index to its node's weight
	*
	* Add a promotable clone instance's sort index (which sums its promotion
	* preferences and scores of relevant location constraints for the promoted
	* role) to the node weight of the instance's allocated node.
	*
	* \param[in] data Promotable clone instance
	* \param[in] user_data Clone parent of \p data
	*/
	static void
	add_sort_index_to_node_weight(gpointer data, gpointer user_data)
	{
	pe_resource_t child = (pe_resource_t ) data;
	pe_resource_t clone = (pe_resource_t ) user_data;

	pe_node_t *node = NULL;
	pe_node_t *chosen = NULL;

	if (child->sort_index < 0) {
	pe_rsc_trace(clone, "Not adding sort index of %s: negative", child->id);
	return;
	}

	chosen = child->fns->location(child, NULL, FALSE);
	if (chosen == NULL) {
	pe_rsc_trace(clone, "Not adding sort index of %s: inactive", child->id);
	return;
	}

	node = (pe_node_t *) pe_hash_table_lookup(clone->allowed_nodes,
	chosen->details->id);
	CRM_ASSERT(node != NULL);

	pe_rsc_trace(clone, "Adding sort index %s of %s to weight for %s",
	pcmk_readable_score(child->sort_index), child->id,
	node->details->uname);
	node->weight = pcmk__add_scores(child->sort_index, node->weight);
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent's node weights if for promoted role
	*
	* \param[in] data Colocation constraint to apply
	* \param[in] user_data Promotable clone that is constraint's dependent
	*/
	static void
	apply_coloc_to_dependent(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t constraint = (pcmk__colocation_t ) data;
	pe_resource_t clone = (pe_resource_t ) user_data;
	enum pe_weights flags = 0;

	if (constraint->dependent_role != RSC_ROLE_PROMOTED) {
	return;
	}
	if (constraint->score < INFINITY) {
	flags = pe_weights_rollback;
	}
	pe_rsc_trace(clone, "RHS: %s with %s: %d",
	constraint->dependent->id, constraint->primary->id,
	constraint->score);
	pcmk__apply_colocation(constraint, clone, constraint->primary, flags);
	}

	/*!
	* \internal
	* \brief Apply colocation to primary's node weights if for promoted role
	*
	* \param[in] data Colocation constraint to apply
	* \param[in] user_data Promotable clone that is constraint's primary
	*/
	static void
	apply_coloc_to_primary(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t constraint = (pcmk__colocation_t ) data;
	pe_resource_t clone = (pe_resource_t ) user_data;

	if ((constraint->primary_role != RSC_ROLE_PROMOTED)
	\|\| !pcmk__colocation_has_influence(constraint, NULL)) {
	return;
	}

	pe_rsc_trace(clone, "LHS: %s with %s: %d",
	constraint->dependent->id, constraint->primary->id,
	constraint->score);
	pcmk__apply_colocation(constraint, clone, constraint->dependent,
	pe_weights_rollback\|pe_weights_positive);
	}

	/*!
	* \internal
	* \brief Set clone instance's sort index to its node's weight
	*
	* \param[in] data Promotable clone instance
	* \param[in] user_data Parent clone of \p data
	*/
	static void
	set_sort_index_to_node_weight(gpointer data, gpointer user_data)
	{
	pe_resource_t child = (pe_resource_t ) data;
	pe_resource_t clone = (pe_resource_t ) user_data;

	pe_node_t *chosen = child->fns->location(child, NULL, FALSE);

	if (!pcmk_is_set(child->flags, pe_rsc_managed)
	&& (child->next_role == RSC_ROLE_PROMOTED)) {
	child->sort_index = INFINITY;
	pe_rsc_trace(clone,
	"Final sort index for %s is INFINITY (unmanaged promoted)",
	child->id);

	} else if ((chosen == NULL) \|\| (child->sort_index < 0)) {
	pe_rsc_trace(clone,
	"Final sort index for %s is %d (ignoring node weight)",
	child->id, child->sort_index);

	} else {
	pe_node_t *node = NULL;

	node = (pe_node_t *) pe_hash_table_lookup(clone->allowed_nodes,
	chosen->details->id);
	CRM_ASSERT(node != NULL);

	child->sort_index = node->weight;
	pe_rsc_trace(clone,
	"Merging weights for %s: final sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	}

	/*!
	* \internal
	* \brief Sort a promotable clone's instances by descending promotion priority
	*
	* \param[in] clone Promotable clone to sort
	*/
	static void
	sort_promotable_instances(pe_resource_t *clone)
	{
	if (pe__set_clone_flag(clone, pe__clone_promotion_constrained)
	== pcmk_rc_already) {
	return;
	}
	pe__set_resource_flags(clone, pe_rsc_merging);

	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	pe_rsc_trace(clone,
	"Merging weights for %s: initial sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	pe__show_node_weights(true, clone, "Before", clone->allowed_nodes,
	clone->cluster);

	g_list_foreach(clone->children, add_sort_index_to_node_weight, clone);
	g_list_foreach(clone->rsc_cons, apply_coloc_to_dependent, clone);
	g_list_foreach(clone->rsc_cons_lhs, apply_coloc_to_primary, clone);

	// Ban resource from all nodes if it needs a ticket but doesn't have it
	pcmk__require_promotion_tickets(clone);

	pe__show_node_weights(true, clone, "After", clone->allowed_nodes,
	clone->cluster);

	// Reset sort indexes to final node weights
	g_list_foreach(clone->children, set_sort_index_to_node_weight, clone);

	// Finally, sort instances in descending order of promotion priority
	clone->children = g_list_sort(clone->children, cmp_promotable_instance);
	pe__clear_resource_flags(clone, pe_rsc_merging);
	}

	/*!
	* \internal
	* \brief Find the active instance (if any) of an anonymous clone on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return
	*/
	static pe_resource_t *
	find_active_anon_instance(pe_resource_t clone, const char id,
	const pe_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pe_resource_t *child = iter->data;
	pe_resource_t *active = NULL;

	// Use ->find_rsc() in case this is a cloned group
	active = clone->fns->find_rsc(child, id, node,
	pe_find_clone\|pe_find_current);
	if (active != NULL) {
	return active;
	}
	}
	return NULL;
	}

	/*
	* \brief Check whether an anonymous clone instance is known on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return true if \p id instance of \p clone is known on \p node,
	* otherwise false
	*/
	static bool
	anonymous_known_on(const pe_resource_t clone, const char id,
	const pe_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pe_resource_t *child = iter->data;

	/* Use ->find_rsc() because this might be a cloned group, and knowing
	* that other members of the group are known here implies nothing.
	*/
	child = clone->fns->find_rsc(child, id, NULL, pe_find_clone);
	CRM_LOG_ASSERT(child != NULL);
	if (child != NULL) {
	if (g_hash_table_lookup(child->known_on, node->details->id)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is allowed to run \p rsc, otherwise false
	*/
	static bool
	is_allowed(const pe_resource_t rsc, const pe_node_t node)
	{
	pe_node_t *allowed = pe_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	return (allowed != NULL) && (allowed->weight >= 0);
	}

	/*!
	* \brief Check whether a clone instance's promotion score should be considered
	*
	* \param[in] rsc Promotable clone instance to check
	* \param[in] node Node where score would be applied
	*
	* \return true if \p rsc's promotion score should be considered on \p node,
	* otherwise false
	*/
	static bool
	promotion_score_applies(pe_resource_t rsc, const pe_node_t node)
	{
	char *id = clone_strip(rsc->id);
	pe_resource_t *parent = uber_parent(rsc);
	pe_resource_t *active = NULL;
	const char *reason = "allowed";

	// Some checks apply only to anonymous clone instances
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {

	// If instance is active on the node, its score definitely applies
	active = find_active_anon_instance(parent, id, node);
	if (active == rsc) {
	reason = "active";
	goto check_allowed;
	}

	/* If no instance is active on this node, this instance's score will
	* count if it has been probed on this node.
	*/
	if ((active == NULL) && anonymous_known_on(parent, id, node)) {
	reason = "probed";
	goto check_allowed;
	}
	}

	/* If this clone's status is unknown on all nodes (e.g. cluster startup),
	* take all instances' scores into account, to make sure we use any
	* permanent promotion scores.
	*/
	if ((rsc->running_on == NULL) && (g_hash_table_size(rsc->known_on) == 0)) {
	reason = "none probed";
	goto check_allowed;
	}

	/* Otherwise, we've probed and/or started the resource somewhere, so
	* consider promotion scores on nodes where we know the status.
	*/
	if ((pe_hash_table_lookup(rsc->known_on, node->details->id) != NULL)
	\|\| (pe_find_node_id(rsc->running_on, node->details->id) != NULL)) {
	reason = "known";
	} else {
	pe_rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: not probed",
	rsc->id, id, node->details->uname);
	free(id);
	return false;
	}

	check_allowed:
	if (is_allowed(rsc, node)) {
	pe_rsc_trace(rsc, "Counting %s promotion score (for %s) on %s: %s",
	rsc->id, id, node->details->uname, reason);
	free(id);
	return true;
	}

	pe_rsc_trace(rsc, "Ignoring %s promotion score (for %s) on %s: not allowed",
	rsc->id, id, node->details->uname);
	free(id);
	return false;
	}

	/*!
	* \internal
	* \brief Get the value of a promotion score node attribute
	*
	* \param[in] rsc Promotable clone instance to get promotion score for
	* \param[in] node Node to get promotion score for
	* \param[in] name Resource name to use in promotion score attribute name
	*
	* \return Value of promotion score node attribute for \p rsc on \p node
	*/
	static const char *
	promotion_attr_value(pe_resource_t rsc, const pe_node_t node,
	const char *name)
	{
	char *attr_name = NULL;
	const char *attr_value = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (name != NULL), return NULL);

	attr_name = pcmk_promotion_score_name(name);
	attr_value = pe_node_attribute_calculated(node, attr_name, rsc);
	free(attr_name);
	return attr_value;
	}

	/*!
	* \internal
	* \brief Get the promotion score for a clone instance on a node
	*
	* \param[in] rsc Promotable clone instance to get score for
	* \param[in] node Node to get score for
	* \param[out] is_default If non-NULL, will be set true if no score available
	*
	* \return Promotion score for \p rsc on \p node (or 0 if none)
	*/
	static int
	promotion_score(pe_resource_t rsc, const pe_node_t node, bool *is_default)
	{
	char *name = NULL;
	const char *attr_value = NULL;

	if (is_default != NULL) {
	*is_default = true;
	}

	CRM_CHECK((rsc != NULL) && (node != NULL), return 0);

	/* If this is an instance of a cloned group, the promotion score is the sum
	* of all members' promotion scores.
	*/
	if (rsc->children != NULL) {
	int score = 0;

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;
	bool child_default = false;
	int child_score = promotion_score(child, node, &child_default);

	if (!child_default && (is_default != NULL)) {
	*is_default = false;
	}
	score += child_score;
	}
	return score;
	}

	if (!promotion_score_applies(rsc, node)) {
	return 0;
	}

	/* For the promotion score attribute name, use the name the resource is
	* known as in resource history, since that's what crm_attribute --promotion
	* would have used.
	*/
	name = (rsc->clone_name == NULL)? rsc->id : rsc->clone_name;

	attr_value = promotion_attr_value(rsc, node, name);
	if (attr_value != NULL) {
	pe_rsc_trace(rsc, "Promotion score for %s on %s = %s",
	name, node->details->uname, pcmk__s(attr_value, "(unset)"));
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	/* If we don't have any resource history yet, we won't have clone_name.
	* In that case, for anonymous clones, try the resource name without
	* any instance number.
	*/
	name = clone_strip(rsc->id);
	if (strcmp(rsc->id, name) != 0) {
	attr_value = promotion_attr_value(rsc, node, name);
	pe_rsc_trace(rsc, "Promotion score for %s on %s (for %s) = %s",
	name, node->details->uname, rsc->id,
	pcmk__s(attr_value, "(unset)"));
	}
	free(name);
	}

	if (attr_value == NULL) {
	return 0;
	}

	if (is_default != NULL) {
	*is_default = false;
	}
	return char2score(attr_value);
	}

	+/*!
	+ * \internal
	+ * \brief Include promotion scores in instances' node weights and priorities
	+ *
	+ * \param[in] rsc Promotable clone resource to update
	+ */
	void
	pcmk__add_promotion_scores(pe_resource_t *rsc)
	{
	- int score, new_score;
	- GList *gIter = rsc->children;
	-
	if (pe__set_clone_flag(rsc, pe__clone_promotion_added) == pcmk_rc_already) {
	return;
	}

	- for (; gIter != NULL; gIter = gIter->next) {
	+ for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	+ pe_resource_t child_rsc = (pe_resource_t ) iter->data;
	+
	GHashTableIter iter;
	pe_node_t *node = NULL;
	- pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	+ int score, new_score;

	g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
	- while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	+ while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, false, false)) {
	/* This node will never be promoted, so don't apply the
	* promotion score, as that may lead to clone shuffling.
	*/
	continue;
	}

	score = promotion_score(child_rsc, node, NULL);
	if (score > 0) {
	new_score = pcmk__add_scores(node->weight, score);
	if (new_score != node->weight) {
	- pe_rsc_trace(rsc, "\t%s: Updating preference for %s (%d->%d)",
	- child_rsc->id, node->details->uname, node->weight, new_score);
	+ pe_rsc_trace(rsc,
	+ "Adding promotion score to preference "
	+ "for %s on %s (%d->%d)",
	+ child_rsc->id, node->details->uname,
	+ node->weight, new_score);
	node->weight = new_score;
	}
	}

	- new_score = QB_MAX(child_rsc->priority, score);
	- if (new_score != child_rsc->priority) {
	- pe_rsc_trace(rsc, "\t%s: Updating priority (%d->%d)",
	- child_rsc->id, child_rsc->priority, new_score);
	- child_rsc->priority = new_score;
	+ if (score > child_rsc->priority) {
	+ pe_rsc_trace(rsc,
	+ "Updating %s priority to promotion score (%d->%d)",
	+ child_rsc->id, child_rsc->priority, score);
	+ child_rsc->priority = score;
	}
	}
	}
	}

	+/*!
	+ * \internal
	+ * \brief If a resource's current role is started, change it to unpromoted
	+ *
	+ * \param[in] data Resource to update
	+ * \param[in] user_data Ignored
	+ */
	static void
	-set_role_unpromoted(pe_resource_t *rsc, bool current)
	+set_current_role_unpromoted(void data, void user_data)
	{
	- GList *gIter = rsc->children;
	+ pe_resource_t rsc = (pe_resource_t ) data;

	- if (current) {
	- if (rsc->role == RSC_ROLE_STARTED) {
	- rsc->role = RSC_ROLE_UNPROMOTED;
	- }
	-
	- } else {
	- GList *allocated = NULL;
	-
	- rsc->fns->location(rsc, &allocated, FALSE);
	- pe__set_next_role(rsc, (allocated? RSC_ROLE_UNPROMOTED : RSC_ROLE_STOPPED),
	- "unpromoted instance");
	- g_list_free(allocated);
	+ if (rsc->role == RSC_ROLE_STARTED) {
	+ // Promotable clones should use unpromoted role instead of started
	+ rsc->role = RSC_ROLE_UNPROMOTED;
	}
	+ g_list_foreach(rsc->children, set_current_role_unpromoted, NULL);
	+}

	- for (; gIter != NULL; gIter = gIter->next) {
	- pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	+/*!
	+ * \internal
	+ * \brief Set a resource's next role to unpromoted (or stopped if unassigned)
	+ *
	+ * \param[in] data Resource to update
	+ * \param[in] user_data Ignored
	+ */
	+static void
	+set_next_role_unpromoted(void data, void user_data)
	+{
	+ pe_resource_t rsc = (pe_resource_t ) data;
	+ GList *assigned = NULL;

	- set_role_unpromoted(child_rsc, current);
	+ rsc->fns->location(rsc, &assigned, FALSE);
	+ if (assigned == NULL) {
	+ pe__set_next_role(rsc, RSC_ROLE_STOPPED, "stopped instance");
	+ } else {
	+ pe__set_next_role(rsc, RSC_ROLE_UNPROMOTED, "unpromoted instance");
	+ g_list_free(assigned);
	}
	+ g_list_foreach(rsc->children, set_next_role_unpromoted, NULL);
	}

	+/*!
	+ * \internal
	+ * \brief Set a resource's next role to promoted if not already set
	+ *
	+ * \param[in] data Resource to update
	+ * \param[in] user_data Ignored
	+ */
	static void
	-set_role_promoted(pe_resource_t *rsc, gpointer user_data)
	+set_next_role_promoted(void *data, gpointer user_data)
	{
	+ pe_resource_t rsc = (pe_resource_t ) data;
	+
	if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	pe__set_next_role(rsc, RSC_ROLE_PROMOTED, "promoted instance");
	}
	-
	- g_list_foreach(rsc->children, (GFunc) set_role_promoted, NULL);
	+ g_list_foreach(rsc->children, set_next_role_promoted, NULL);
	}

	-pe_node_t *
	-pcmk__set_instance_roles(pe_resource_t rsc, pe_working_set_t data_set)
	+/*!
	+ * \internal
	+ * \brief Show instance's promotion score on node where it will be active
	+ *
	+ * \param[in] instance Promotable clone instance to show
	+ */
	+static void
	+show_promotion_score(pe_resource_t *instance)
	{
	- int promoted = 0;
	- GList *gIter = NULL;
	- GList *gIter2 = NULL;
	- GHashTableIter iter;
	- pe_node_t *node = NULL;
	- pe_node_t *chosen = NULL;
	- enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
	- clone_variant_data_t *clone_data = NULL;
	+ pe_node_t *chosen = instance->fns->location(instance, NULL, FALSE);

	- get_clone_variant_data(clone_data, rsc);
	+ if (pcmk_is_set(instance->cluster->flags, pe_flag_show_scores)
	+ && !pcmk__is_daemon && (instance->cluster->priv != NULL)) {

	- // Repurpose count to track the number of promoted instances allocated
	- g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	- while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	- node->count = 0;
	- }
	+ pcmk__output_t *out = instance->cluster->priv;

	- /*
	- * assign priority
	- */
	- for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	- GList *list = NULL;
	- pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	+ out->message(out, "promotion-score", instance, chosen,
	+ pcmk_readable_score(instance->sort_index));
	+ } else {
	+ pe_rsc_debug(uber_parent(instance),
	+ "%s promotion score on %s: sort=%s priority=%s",
	+ instance->id,
	+ ((chosen == NULL)? "none" : chosen->details->uname),
	+ pcmk_readable_score(instance->sort_index),
	+ pcmk_readable_score(instance->priority));
	+ }
	+}

	- pe_rsc_trace(rsc, "Assigning priority for %s: %s", child_rsc->id,
	- role2text(child_rsc->next_role));
	+/*!
	+ * \internal
	+ * \brief Set a clone instance's promotion priority
	+ *
	+ * \param[in] data Promotable clone instance to update
	+ * \param[in] user_data Instance's parent clone
	+ */
	+static void
	+set_instance_priority(gpointer data, gpointer user_data)
	+{
	+ pe_resource_t instance = (pe_resource_t ) data;
	+ pe_resource_t clone = (pe_resource_t ) user_data;
	+ pe_node_t *chosen = NULL;
	+ enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
	+ GList *list = NULL;

	- if (child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) {
	- set_role_unpromoted(child_rsc, true);
	- }
	+ pe_rsc_trace(clone, "Assigning priority for %s: %s", instance->id,
	+ role2text(instance->next_role));

	- chosen = child_rsc->fns->location(child_rsc, &list, FALSE);
	- if (pcmk__list_of_multiple(list)) {
	- pcmk__config_err("Cannot promote non-colocated child %s",
	- child_rsc->id);
	- }
	+ if (instance->fns->state(instance, TRUE) == RSC_ROLE_STARTED) {
	+ set_current_role_unpromoted(instance, NULL);
	+ }

	- g_list_free(list);
	- if (chosen == NULL) {
	- continue;
	- }
	+ // Only an instance that will be active can be promoted
	+ chosen = instance->fns->location(instance, &list, FALSE);
	+ if (pcmk__list_of_multiple(list)) {
	+ pcmk__config_err("Cannot promote non-colocated child %s",
	+ instance->id);
	+ }
	+ g_list_free(list);
	+ if (chosen == NULL) {
	+ return;
	+ }

	- next_role = child_rsc->fns->state(child_rsc, FALSE);
	- switch (next_role) {
	- case RSC_ROLE_STARTED:
	- case RSC_ROLE_UNKNOWN:
	- {
	- bool is_default = false;
	-
	- child_rsc->priority = promotion_score(child_rsc, chosen,
	- &is_default);
	- if (is_default) {
	- /*
	- * Default to -1 if no value is set. This allows
	- * instances eligible for promotion to be specified
	- * based solely on rsc_location constraints, but
	- * prevents any instance from being promoted if neither
	- * a constraint nor a promotion score is present
	- */
	- child_rsc->priority = -1;
	- }
	+ next_role = instance->fns->state(instance, FALSE);
	+ switch (next_role) {
	+ case RSC_ROLE_STARTED:
	+ case RSC_ROLE_UNKNOWN:
	+ // Set instance priority to its promotion score (or -1 if none)
	+ {
	+ bool is_default = false;
	+
	+ instance->priority = promotion_score(instance, chosen,
	+ &is_default);
	+ if (is_default) {
	+ /*
	+ * Default to -1 if no value is set. This allows
	+ * instances eligible for promotion to be specified
	+ * based solely on rsc_location constraints, but
	+ * prevents any instance from being promoted if neither
	+ * a constraint nor a promotion score is present
	+ */
	+ instance->priority = -1;
	}
	- break;
	-
	- case RSC_ROLE_UNPROMOTED:
	- case RSC_ROLE_STOPPED:
	- child_rsc->priority = -INFINITY;
	- break;
	- case RSC_ROLE_PROMOTED:
	- /* We will arrive here if we're re-creating actions after a stonith
	- */
	- break;
	- default:
	- CRM_CHECK(FALSE /* unhandled */ ,
	- crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id));
	- }
	+ }
	+ break;

	- apply_promoted_locations(child_rsc, child_rsc->rsc_location, chosen);
	- apply_promoted_locations(child_rsc, rsc->rsc_location, chosen);
	+ case RSC_ROLE_UNPROMOTED:
	+ case RSC_ROLE_STOPPED:
	+ // Instance can't be promoted
	+ instance->priority = -INFINITY;
	+ break;

	- for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) {
	- pcmk__colocation_t cons = (pcmk__colocation_t ) gIter2->data;
	+ case RSC_ROLE_PROMOTED:
	+ // Nothing needed (re-creating actions after scheduling fencing)
	+ break;

	- child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->primary, cons,
	- data_set);
	- }
	+ default:
	+ CRM_CHECK(FALSE, crm_err("Unknown resource role %d for %s",
	+ next_role, instance->id));
	+ }

	- child_rsc->sort_index = child_rsc->priority;
	- pe_rsc_trace(rsc, "Assigning priority for %s: %d", child_rsc->id, child_rsc->priority);
	+ // Add relevant location constraint scores for promoted role
	+ apply_promoted_locations(instance, instance->rsc_location, chosen);
	+ apply_promoted_locations(instance, clone->rsc_location, chosen);

	- if (next_role == RSC_ROLE_PROMOTED) {
	- child_rsc->sort_index = INFINITY;
	- }
	+ // Apply relevant colocations with promoted role
	+ for (GList *iter = instance->rsc_cons; iter != NULL; iter = iter->next) {
	+ pcmk__colocation_t cons = (pcmk__colocation_t ) iter->data;
	+
	+ instance->cmds->rsc_colocation_lh(instance, cons->primary, cons,
	+ instance->cluster);
	}

	- sort_promotable_instances(rsc);
	+ instance->sort_index = instance->priority;
	+ if (next_role == RSC_ROLE_PROMOTED) {
	+ instance->sort_index = INFINITY;
	+ }
	+ pe_rsc_trace(clone, "Assigning %s priority = %d",
	+ instance->id, instance->priority);
	+}

	- // Choose the first N eligible instances to be promoted
	- for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	- pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	+/*!
	+ * \internal
	+ * \brief Set a promotable clone instance's role
	+ *
	+ * \param[in] data Promotable clone instance to update
	+ * \param[in] user_data Pointer to count of instances chosen for promotion
	+ */
	+static void
	+set_instance_role(gpointer data, gpointer user_data)
	+{
	+ pe_resource_t instance = (pe_resource_t ) data;
	+ int count = (int ) user_data;

	- chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	- if (pcmk_is_set(data_set->flags, pe_flag_show_scores) && !pcmk__is_daemon) {
	- if (data_set->priv != NULL) {
	- pcmk__output_t *out = data_set->priv;
	- out->message(out, "promotion-score", child_rsc, chosen,
	- pcmk_readable_score(child_rsc->sort_index));
	- }
	+ pe_resource_t *clone = uber_parent(instance);
	+ pe_node_t *chosen = NULL;

	- } else {
	- pe_rsc_trace(rsc, "%s promotion score on %s: %s", child_rsc->id,
	- (chosen? chosen->details->uname : "none"),
	- pcmk_readable_score(child_rsc->sort_index));
	- }
	+ show_promotion_score(instance);

	- chosen = NULL; /* nuke 'chosen' so that we don't promote more than the
	- * required number of instances
	- */
	+ if (instance->sort_index < 0) {
	+ pe_rsc_trace(clone, "Not supposed to promote instance %s",
	+ instance->id);

	- if (child_rsc->sort_index < 0) {
	- pe_rsc_trace(rsc, "Not supposed to promote child: %s", child_rsc->id);
	+ } else if ((*count < pe__clone_promoted_max(instance))
	+ \|\| !pcmk_is_set(clone->flags, pe_rsc_managed)) {
	+ chosen = node_to_be_promoted_on(instance);
	+ }

	- } else if ((promoted < clone_data->promoted_max)
	- \|\| !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	- chosen = node_to_be_promoted_on(child_rsc);
	- }
	+ if (chosen == NULL) {
	+ set_next_role_unpromoted(instance, NULL);
	+ return;
	+ }

	- pe_rsc_debug(rsc, "%s promotion score: %d", child_rsc->id, child_rsc->priority);
	+ if ((instance->role < RSC_ROLE_PROMOTED)
	+ && !pcmk_is_set(instance->cluster->flags, pe_flag_have_quorum)
	+ && (instance->cluster->no_quorum_policy == no_quorum_freeze)) {
	+ crm_notice("Clone instance %s cannot be promoted without quorum",
	+ instance->id);
	+ set_next_role_unpromoted(instance, NULL);
	+ return;
	+ }

	- if (chosen == NULL) {
	- set_role_unpromoted(child_rsc, false);
	- continue;
	+ chosen->count++;
	+ pe_rsc_info(clone, "Choosing %s (%s) on %s for promotion",
	+ instance->id, role2text(instance->role),
	+ chosen->details->uname);
	+ set_next_role_promoted(instance, NULL);
	+ (*count)++;
	+}

	- } else if ((child_rsc->role < RSC_ROLE_PROMOTED)
	- && !pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	- && data_set->no_quorum_policy == no_quorum_freeze) {
	- crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	- child_rsc->id, role2text(child_rsc->role), role2text(child_rsc->next_role));
	- set_role_unpromoted(child_rsc, false);
	- continue;
	- }
	+/*!
	+ * \internal
	+ * \brief Set roles for all instances of a promotable clone
	+ *
	+ * \param[in] clone Promotable clone resource to update
	+ */
	+void
	+pcmk__set_instance_roles(pe_resource_t *rsc)
	+{
	+ int promoted = 0;
	+ GHashTableIter iter;
	+ pe_node_t *node = NULL;

	- chosen->count++;
	- pe_rsc_info(rsc, "Promoting %s (%s %s)",
	- child_rsc->id, role2text(child_rsc->role), chosen->details->uname);
	- set_role_promoted(child_rsc, NULL);
	- promoted++;
	+ // Repurpose count to track the number of promoted instances allocated
	+ g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	+ while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	+ node->count = 0;
	}

	- pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	- rsc->id, promoted, clone_data->promoted_max);
	+ // Set instances' promotion priorities and sort by highest priority first
	+ g_list_foreach(rsc->children, set_instance_priority, rsc);
	+ sort_promotable_instances(rsc);

	- return NULL;
	+ // Choose the first N eligible instances to be promoted
	+ g_list_foreach(rsc->children, set_instance_role, &promoted);
	+ pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	+ rsc->id, promoted, pe__clone_promoted_max(rsc));
	}

	void
	create_promotable_actions(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_action_t *action = NULL;
	GList *gIter = rsc->children;
	pe_action_t *action_complete = NULL;
	bool any_promoting = false;
	bool any_demoting = false;

	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_debug(rsc, "Creating actions for %s", rsc->id);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Creating actions for %s", child_rsc->id);
	child_rsc->cmds->create_actions(child_rsc, data_set);
	check_for_role_change(child_rsc, &any_demoting, &any_promoting);
	}

	/* promote */
	action = pcmk__new_rsc_pseudo_action(rsc, RSC_PROMOTE, !any_promoting,
	true);
	action_complete = pcmk__new_rsc_pseudo_action(rsc, RSC_PROMOTED,
	!any_promoting, true);
	action_complete->priority = INFINITY;

	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify = pcmk__clone_notif_pseudo_ops(rsc,
	RSC_PROMOTE,
	action,
	action_complete);
	}

	/* demote */
	action = pcmk__new_rsc_pseudo_action(rsc, RSC_DEMOTE, !any_demoting, true);
	action_complete = pcmk__new_rsc_pseudo_action(rsc, RSC_DEMOTED,
	!any_demoting, true);
	action_complete->priority = INFINITY;

	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify = pcmk__clone_notif_pseudo_ops(rsc,
	RSC_DEMOTE,
	action,
	action_complete);

	if (clone_data->promote_notify) {
	/* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day
	* Requires exposing *_notify
	*/
	order_actions(clone_data->stop_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->start_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre,
	pe_order_optional);
	}
	}

	/* restore the correct priority */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->priority = rsc->priority;
	}
	}

	void
	promote_demote_constraints(pe_resource_t rsc, pe_working_set_t data_set)
	{
	/* global stopped before start */
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_START,
	pe_order_optional, data_set);

	/* global stopped before promote */
	pcmk__order_resource_actions(rsc, RSC_STOPPED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);

	/* global demoted before start */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_START,
	pe_order_optional, data_set);

	/* global started before promote */
	pcmk__order_resource_actions(rsc, RSC_STARTED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);

	/* global demoted before stop */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_STOP,
	pe_order_optional, data_set);

	/* global demote before demoted */
	pcmk__order_resource_actions(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED,
	pe_order_optional, data_set);

	/* global demoted before promote */
	pcmk__order_resource_actions(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE,
	pe_order_optional, data_set);
	}


	void
	promotable_constraints(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GList *gIter = rsc->children;
	pe_resource_t *last_rsc = NULL;

	promote_demote_constraints(rsc, data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	/* child demote before promote */
	pcmk__order_resource_actions(child_rsc, RSC_DEMOTE, child_rsc,
	RSC_PROMOTE, pe_order_optional, data_set);

	order_instance_promotion(rsc, child_rsc, last_rsc);
	order_instance_demotion(rsc, child_rsc, last_rsc);

	last_rsc = child_rsc;
	}
	}

	static void
	node_hash_update_one(GHashTable * hash, pe_node_t * other, const char *attr, int score)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;
	const char *value = NULL;

	if (other == NULL) {
	return;

	} else if (attr == NULL) {
	attr = CRM_ATTR_UNAME;
	}

	value = pe_node_attribute_raw(other, attr);
	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	const char *tmp = pe_node_attribute_raw(node, attr);

	if (pcmk__str_eq(value, tmp, pcmk__str_casei)) {
	crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight);
	node->weight = pcmk__add_scores(node->weight, score);
	}
	}
	}

	void
	promotable_colocation_rh(pe_resource_t dependent, pe_resource_t primary,
	pcmk__colocation_t *constraint,
	pe_working_set_t *data_set)
	{
	GList *gIter = NULL;

	if (pcmk_is_set(dependent->flags, pe_rsc_provisional)) {
	GList *affected_nodes = NULL;

	for (gIter = primary->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	pe_node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE);

	pe_rsc_trace(primary, "Processing: %s", child_rsc->id);
	if ((chosen != NULL) && (next_role == constraint->primary_role)) {
	pe_rsc_trace(primary, "Applying: %s %s %s %d", child_rsc->id,
	role2text(next_role), chosen->details->uname, constraint->score);
	if (constraint->score < INFINITY) {
	node_hash_update_one(dependent->allowed_nodes, chosen,
	constraint->node_attribute, constraint->score);
	}
	affected_nodes = g_list_prepend(affected_nodes, chosen);
	}
	}

	/* Only do this if it's not a promoted-with-promoted colocation. Doing
	* this unconditionally would prevent unpromoted instances from being
	* started.
	*/
	if ((constraint->dependent_role != RSC_ROLE_PROMOTED)
	\|\| (constraint->primary_role != RSC_ROLE_PROMOTED)) {

	if (constraint->score >= INFINITY) {
	node_list_exclude(dependent->allowed_nodes, affected_nodes,
	TRUE);
	}
	}
	g_list_free(affected_nodes);

	} else if (constraint->dependent_role == RSC_ROLE_PROMOTED) {
	pe_resource_t *primary_instance;

	primary_instance = find_compatible_child(dependent, primary,
	constraint->primary_role,
	FALSE, data_set);
	if ((primary_instance == NULL) && (constraint->score >= INFINITY)) {
	pe_rsc_trace(dependent, "%s can't be promoted %s",
	dependent->id, constraint->id);
	dependent->priority = -INFINITY;

	} else if (primary_instance != NULL) {
	int new_priority = pcmk__add_scores(dependent->priority,
	constraint->score);

	pe_rsc_debug(dependent, "Applying %s to %s",
	constraint->id, dependent->id);
	pe_rsc_debug(dependent, "\t%s: %d->%d",
	dependent->id, dependent->priority, new_priority);
	dependent->priority = new_priority;
	}
	}

	return;
	}
	diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
	index bac0cc49bf..ecec151087 100644
	--- a/lib/pengine/clone.c
	+++ b/lib/pengine/clone.c
	@@ -1,1189 +1,1223 @@
	/*
	* Copyright 2004-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>

	#define VARIANT_CLONE 1
	#include "./variant.h"

	#ifdef PCMK__COMPAT_2_0
	#define PROMOTED_INSTANCES RSC_ROLE_PROMOTED_LEGACY_S "s"
	#define UNPROMOTED_INSTANCES RSC_ROLE_UNPROMOTED_LEGACY_S "s"
	#else
	#define PROMOTED_INSTANCES RSC_ROLE_PROMOTED_S
	#define UNPROMOTED_INSTANCES RSC_ROLE_UNPROMOTED_S
	#endif

	+/*!
	+ * \internal
	+ * \brief Return the maximum number of clone instances allowed to be promoted
	+ *
	+ * \param[in] clone Promotable clone or clone instance to check
	+ *
	+ * \return Maximum promoted instances for \p clone
	+ */
	+int
	+pe__clone_promoted_max(pe_resource_t *clone)
	+{
	+ clone_variant_data_t *clone_data = NULL;
	+
	+ get_clone_variant_data(clone_data, uber_parent(clone));
	+ return clone_data->promoted_max;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Return the maximum number of clone instances allowed to be promoted
	+ *
	+ * \param[in] clone Promotable clone or clone instance to check
	+ *
	+ * \return Maximum promoted instances for \p clone
	+ */
	+int
	+pe__clone_promoted_node_max(pe_resource_t *clone)
	+{
	+ clone_variant_data_t *clone_data = NULL;
	+
	+ get_clone_variant_data(clone_data, uber_parent(clone));
	+ return clone_data->promoted_node_max;
	+}
	+
	static GList *
	sorted_hash_table_values(GHashTable *table)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
	retval = g_list_prepend(retval, (char *) value);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) strcmp);
	return retval;
	}

	static GList *
	nodes_with_status(GHashTable table, const char status)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!strcmp((char *) value, status)) {
	retval = g_list_prepend(retval, key);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
	return retval;
	}

	static char *
	node_list_to_str(GList *list)
	{
	char *retval = NULL;
	size_t len = 0;

	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__add_word(&retval, &len, (char *) iter->data);
	}

	return retval;
	}

	static void
	clone_header(pcmk__output_t out, int rc, pe_resource_t rsc, clone_variant_data_t clone_data)
	{
	char *attrs = NULL;
	size_t len = 0;

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pcmk__add_separated_word(&attrs, &len, "promotable", ", ");
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	pcmk__add_separated_word(&attrs, &len, "unique", ", ");
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pcmk__add_separated_word(&attrs, &len, "unmanaged", ", ");
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, &len, "disabled", ", ");
	}

	if (attrs) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s] (%s)",
	rsc->id, ID(clone_data->xml_obj_child),
	attrs);
	free(attrs);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s]",
	rsc->id, ID(clone_data->xml_obj_child))
	}
	}

	void
	pe__force_anon(const char standard, pe_resource_t rsc, const char *rid,
	pe_working_set_t *data_set)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_warn("Ignoring " XML_RSC_ATTR_UNIQUE " for %s because %s resources "
	"such as %s can be used only as anonymous clones",
	rsc->id, standard, rid);

	clone_data->clone_node_max = 1;
	clone_data->clone_max = QB_MIN(clone_data->clone_max,
	g_list_length(data_set->nodes));
	}
	}

	pe_resource_t *
	find_clone_instance(pe_resource_t * rsc, const char sub_id, pe_working_set_t data_set)
	{
	char *child_id = NULL;
	pe_resource_t *child = NULL;
	const char *child_base = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	child_base = ID(clone_data->xml_obj_child);
	child_id = crm_strdup_printf("%s:%s", child_base, sub_id);
	child = pe_find_resource(rsc->children, child_id);

	free(child_id);
	return child;
	}

	pe_resource_t *
	pe__create_clone_child(pe_resource_t rsc, pe_working_set_t data_set)
	{
	gboolean as_orphan = FALSE;
	char *inc_num = NULL;
	char *inc_max = NULL;
	pe_resource_t *child_rsc = NULL;
	xmlNode *child_copy = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);

	if (clone_data->total_clones >= clone_data->clone_max) {
	// If we've already used all available instances, this is an orphan
	as_orphan = TRUE;
	}

	// Allocate instance numbers in numerical order (starting at 0)
	inc_num = pcmk__itoa(clone_data->total_clones);
	inc_max = pcmk__itoa(clone_data->clone_max);

	child_copy = copy_xml(clone_data->xml_obj_child);

	crm_xml_add(child_copy, XML_RSC_ATTR_INCARNATION, inc_num);

	if (common_unpack(child_copy, &child_rsc, rsc, data_set) == FALSE) {
	pe_err("Failed unpacking resource %s", crm_element_value(child_copy, XML_ATTR_ID));
	child_rsc = NULL;
	goto bail;
	}
	/* child_rsc->globally_unique = rsc->globally_unique; */

	CRM_ASSERT(child_rsc);
	clone_data->total_clones += 1;
	pe_rsc_trace(child_rsc, "Setting clone attributes for: %s", child_rsc->id);
	rsc->children = g_list_append(rsc->children, child_rsc);
	if (as_orphan) {
	pe__set_resource_flags_recursive(child_rsc, pe_rsc_orphan);
	}

	add_hash_param(child_rsc->meta, XML_RSC_ATTR_INCARNATION_MAX, inc_max);
	pe_rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);

	bail:
	free(inc_num);
	free(inc_max);

	return child_rsc;
	}

	gboolean
	clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	int lpc = 0;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = rsc->xml;
	clone_variant_data_t *clone_data = NULL;

	const char *max_clones = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_MAX);
	const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	clone_data = calloc(1, sizeof(clone_variant_data_t));
	rsc->variant_opaque = clone_data;

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	const char *promoted_max = NULL;
	const char *promoted_node_max = NULL;

	promoted_max = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTED_MAX);
	if (promoted_max == NULL) {
	// @COMPAT deprecated since 2.0.0
	promoted_max = g_hash_table_lookup(rsc->meta,
	PCMK_XE_PROMOTED_MAX_LEGACY);
	}

	promoted_node_max = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTED_NODEMAX);
	if (promoted_node_max == NULL) {
	// @COMPAT deprecated since 2.0.0
	promoted_node_max = g_hash_table_lookup(rsc->meta,
	PCMK_XE_PROMOTED_NODE_MAX_LEGACY);
	}

	// Use 1 as default but 0 for minimum and invalid
	if (promoted_max == NULL) {
	clone_data->promoted_max = 1;
	} else {
	pcmk__scan_min_int(promoted_max, &(clone_data->promoted_max), 0);
	}

	// Use 1 as default but 0 for minimum and invalid
	if (promoted_node_max == NULL) {
	clone_data->promoted_node_max = 1;
	} else {
	pcmk__scan_min_int(promoted_node_max,
	&(clone_data->promoted_node_max), 0);
	}
	}

	// Implied by calloc()
	/* clone_data->xml_obj_child = NULL; */

	// Use 1 as default but 0 for minimum and invalid
	if (max_clones_node == NULL) {
	clone_data->clone_node_max = 1;
	} else {
	pcmk__scan_min_int(max_clones_node, &(clone_data->clone_node_max), 0);
	}

	/* Use number of nodes (but always at least 1, which is handy for crm_verify
	* for a CIB without nodes) as default, but 0 for minimum and invalid
	*/
	if (max_clones == NULL) {
	clone_data->clone_max = QB_MAX(1, g_list_length(data_set->nodes));
	} else {
	pcmk__scan_min_int(max_clones, &(clone_data->clone_max), 0);
	}

	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_ORDERED))) {
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", rsc->id,
	clone_data->flags,
	pe__clone_ordered,
	"pe__clone_ordered");
	}

	if ((rsc->flags & pe_rsc_unique) == 0 && clone_data->clone_node_max > 1) {
	pcmk__config_err("Ignoring " XML_RSC_ATTR_PROMOTED_MAX " for %s "
	"because anonymous clones support only one instance "
	"per node", rsc->id);
	clone_data->clone_node_max = 1;
	}

	pe_rsc_trace(rsc, "Options for %s", rsc->id);
	pe_rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
	pe_rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
	pe_rsc_trace(rsc, "\tClone is unique: %s",
	pe__rsc_bool_str(rsc, pe_rsc_unique));
	pe_rsc_trace(rsc, "\tClone is promotable: %s",
	pe__rsc_bool_str(rsc, pe_rsc_promotable));

	// Clones may contain a single group or primitive
	for (a_child = pcmk__xe_first_child(xml_obj); a_child != NULL;
	a_child = pcmk__xe_next(a_child)) {

	if (pcmk__str_any_of((const char *)a_child->name, XML_CIB_TAG_RESOURCE, XML_CIB_TAG_GROUP, NULL)) {
	clone_data->xml_obj_child = a_child;
	break;
	}
	}

	if (clone_data->xml_obj_child == NULL) {
	pcmk__config_err("%s has nothing to clone", rsc->id);
	return FALSE;
	}

	/*
	* Make clones ever so slightly sticky by default
	*
	* This helps ensure clone instances are not shuffled around the cluster
	* for no benefit in situations when pre-allocation is not appropriate
	*/
	if (g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_STICKINESS) == NULL) {
	add_hash_param(rsc->meta, XML_RSC_ATTR_STICKINESS, "1");
	}

	/* This ensures that the globally-unique value always exists for children to
	* inherit when being unpacked, as well as in resource agents' environment.
	*/
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE,
	pe__rsc_bool_str(rsc, pe_rsc_unique));

	if (clone_data->clone_max <= 0) {
	/* Create one child instance so that unpack_find_resource() will hook up
	* any orphans up to the parent correctly.
	*/
	if (pe__create_clone_child(rsc, data_set) == NULL) {
	return FALSE;
	}

	} else {
	// Create a child instance for each available instance number
	for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
	if (pe__create_clone_child(rsc, data_set) == NULL) {
	return FALSE;
	}
	}
	}

	pe_rsc_trace(rsc, "Added %d children to resource %s...", clone_data->clone_max, rsc->id);
	return TRUE;
	}

	gboolean
	clone_active(pe_resource_t * rsc, gboolean all)
	{
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean child_active = child_rsc->fns->active(child_rsc, all);

	if (all == FALSE && child_active) {
	return TRUE;
	} else if (all && child_active == FALSE) {
	return FALSE;
	}
	}

	if (all) {
	return TRUE;
	} else {
	return FALSE;
	}
	}

	static void
	short_print(char list, const char prefix, const char type, const char suffix, long options, void *print_data)
	{
	if(suffix == NULL) {
	suffix = "";
	}

	if (list) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	status_print("%s%s: [ %s ]%s", prefix, type, list, suffix);

	if (options & pe_print_html) {
	status_print("</li>\n");

	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}

	}
	}

	static const char *
	configured_role_str(pe_resource_t * rsc)
	{
	const char *target_role = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_TARGET_ROLE);

	if ((target_role == NULL) && rsc->children && rsc->children->data) {
	target_role = g_hash_table_lookup(((pe_resource_t*)rsc->children->data)->meta,
	XML_RSC_ATTR_TARGET_ROLE);
	}
	return target_role;
	}

	static enum rsc_role_e
	configured_role(pe_resource_t * rsc)
	{
	const char *target_role = configured_role_str(rsc);

	if (target_role) {
	return text2role(target_role);
	}
	return RSC_ROLE_UNKNOWN;
	}

	static void
	clone_print_xml(pe_resource_t * rsc, const char pre_text, long options, void print_data)
	{
	char *child_text = crm_strdup_printf("%s ", pre_text);
	const char *target_role = configured_role_str(rsc);
	GList *gIter = rsc->children;

	status_print("%s<clone ", pre_text);
	status_print("id=\"%s\" ", rsc->id);
	status_print("multi_state=\"%s\" ",
	pe__rsc_bool_str(rsc, pe_rsc_promotable));
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
	status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pe_rsc_failure_ignored));
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</clone>\n", pre_text);
	free(child_text);
	}

	bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any)
	{
	GList *gIter;
	bool all = !any;

	if (pcmk_is_set(rsc->flags, flag)) {
	if(any) {
	return TRUE;
	}
	} else if(all) {
	return FALSE;
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	if(is_set_recursive(gIter->data, flag, any)) {
	if(any) {
	return TRUE;
	}

	} else if(all) {
	return FALSE;
	}
	}

	if(all) {
	return TRUE;
	}
	return FALSE;
	}

	void
	clone_print(pe_resource_t * rsc, const char pre_text, long options, void print_data)
	{
	char *list_text = NULL;
	char *child_text = NULL;
	char *stopped_list = NULL;
	size_t list_text_len = 0;
	size_t stopped_list_len = 0;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	clone_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_clone_variant_data(clone_data, rsc);

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sClone Set: %s [%s]%s%s%s",
	pre_text ? pre_text : "", rsc->id, ID(clone_data->xml_obj_child),
	pcmk_is_set(rsc->flags, pe_rsc_promotable)? " (promotable)" : "",
	pcmk_is_set(rsc->flags, pe_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pe_rsc_managed)? "" : " (unmanaged)");

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (options & pe_print_clone_details) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active \|\| !pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(options, pe_print_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pe_rsc_orphan)
	&& !pcmk_is_set(options, pe_print_clone_active)) {
	pcmk__add_word(&stopped_list, &stopped_list_len, child_rsc->id);
	}

	} else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
	\|\| is_set_recursive(child_rsc, pe_rsc_managed, FALSE) == FALSE
	\|\| is_set_recursive(child_rsc, pe_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);

	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > RSC_ROLE_UNPROMOTED) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, sort_node_uname);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	pcmk__add_word(&list_text, &list_text_len, host->details->uname);
	active_instances++;
	}

	short_print(list_text, child_text, PROMOTED_INSTANCES, NULL, options,
	print_data);
	g_list_free(promoted_list);
	free(list_text);
	list_text = NULL;
	list_text_len = 0;

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, sort_node_uname);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	pcmk__add_word(&list_text, &list_text_len, host->details->uname);
	active_instances++;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == RSC_ROLE_UNPROMOTED) {
	short_print(list_text, child_text,
	UNPROMOTED_INSTANCES " (target-role)", NULL, options,
	print_data);
	} else {
	short_print(list_text, child_text, UNPROMOTED_INSTANCES, NULL,
	options, print_data);
	}

	} else {
	short_print(list_text, child_text, "Started", NULL, options, print_data);
	}

	g_list_free(started_list);
	free(list_text);
	list_text = NULL;
	list_text_len = 0;

	if (!pcmk_is_set(options, pe_print_clone_active)) {
	const char *state = "Stopped";
	enum rsc_role_e role = configured_role(rsc);

	if (role == RSC_ROLE_STOPPED) {
	state = "Stopped (disabled)";
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped list for non-unique clones */
	free(stopped_list);
	stopped_list = NULL;
	stopped_list_len = 0;

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, sort_node_uname);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pe_node_t node = (pe_node_t )nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL) {
	pcmk__add_word(&stopped_list, &stopped_list_len,
	node->details->uname);
	}
	}
	g_list_free(list);
	}

	short_print(stopped_list, child_text, state, NULL, options, print_data);
	free(stopped_list);
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}

	free(child_text);
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__clone_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GList *gIter = rsc->children;
	GList *all = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	all = g_list_prepend(all, (gpointer) "*");

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	rc = pe__name_and_nvpairs_xml(out, true, "clone", 8,
	"id", rsc->id,
	"multi_state", pe__rsc_bool_str(rsc, pe_rsc_promotable),
	"unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
	"managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
	"disabled", pcmk__btoa(pe__resource_is_disabled(rsc)),
	"failed", pe__rsc_bool_str(rsc, pe_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pe_rsc_failure_ignored),
	"target_role", configured_role_str(rsc));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out);
	}

	g_list_free(all);
	return rc;
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__clone_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GHashTable *stopped = NULL;

	char *list_text = NULL;
	size_t list_text_len = 0;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	get_clone_variant_data(clone_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active \|\| !pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(show_opts, pcmk_show_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pe_rsc_orphan)
	&& !pcmk_is_set(show_opts, pcmk_show_clone_detail)
	&& pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	g_hash_table_insert(stopped, strdup(child_rsc->id), strdup("Stopped"));
	}

	} else if (is_set_recursive(child_rsc, pe_rsc_orphan, TRUE)
	\|\| is_set_recursive(child_rsc, pe_rsc_managed, FALSE) == FALSE
	\|\| is_set_recursive(child_rsc, pe_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pe_node_t *location = child_rsc->fns->location(child_rsc, NULL, TRUE);

	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > RSC_ROLE_UNPROMOTED) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	GList *all = NULL;

	clone_header(out, &rc, rsc, clone_data);

	/* Print every resource that's a child of this clone. */
	all = g_list_prepend(all, (gpointer) "*");
	out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	child_rsc, only_node, all);
	g_list_free(all);
	}
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return pcmk_rc_ok;
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, sort_node_uname);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, &list_text_len, host->details->uname);
	active_instances++;
	}
	g_list_free(promoted_list);

	if (list_text != NULL) {
	clone_header(out, &rc, rsc, clone_data);

	out->list_item(out, NULL, PROMOTED_INSTANCES ": [ %s ]", list_text);
	free(list_text);
	list_text = NULL;
	list_text_len = 0;
	}

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, sort_node_uname);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pe_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, &list_text_len, host->details->uname);
	active_instances++;
	}
	g_list_free(started_list);

	if (list_text != NULL) {
	clone_header(out, &rc, rsc, clone_data);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == RSC_ROLE_UNPROMOTED) {
	out->list_item(out, NULL,
	UNPROMOTED_INSTANCES " (target-role): [ %s ]",
	list_text);
	} else {
	out->list_item(out, NULL, UNPROMOTED_INSTANCES ": [ %s ]",
	list_text);
	}

	} else {
	out->list_item(out, NULL, "Started: [ %s ]", list_text);
	}
	free(list_text);
	list_text = NULL;
	list_text_len = 0;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped table for non-unique clones */
	if (stopped != NULL) {
	g_hash_table_destroy(stopped);
	stopped = NULL;
	}

	if (list == NULL) {
	/* Clusters with symmetrical=false haven't calculated allowed_nodes yet
	* If we've not probed for them yet, the Stopped list will be empty
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, sort_node_uname);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pe_node_t node = (pe_node_t )nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
	pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
	const char *state = "Stopped";

	if (configured_role(rsc) == RSC_ROLE_STOPPED) {
	state = "Stopped (disabled)";
	}

	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	g_hash_table_insert(stopped, strdup(node->details->uname),
	crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
	} else {
	g_hash_table_insert(stopped, strdup(node->details->uname),
	strdup(state));
	}
	}
	}
	g_list_free(list);
	}

	if (stopped != NULL) {
	GList *list = sorted_hash_table_values(stopped);

	clone_header(out, &rc, rsc, clone_data);

	for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
	const char *status = status_iter->data;
	GList *nodes = nodes_with_status(stopped, status);
	char *str = node_list_to_str(nodes);

	if (str != NULL) {
	out->list_item(out, NULL, "%s: [ %s ]", status, str);
	free(str);
	}

	g_list_free(nodes);
	}

	g_list_free(list);
	g_hash_table_destroy(stopped);

	/* If there are no instances of this clone (perhaps because there are no
	* nodes configured), simply output the clone header by itself. This can
	* come up in PCS testing.
	*/
	} else if (active_instances == 0) {
	clone_header(out, &rc, rsc, clone_data);
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	void
	clone_free(pe_resource_t * rsc)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pe_rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	free_xml(child_rsc->xml);
	child_rsc->xml = NULL;
	/* There could be a saved unexpanded xml */
	free_xml(child_rsc->orig_xml);
	child_rsc->orig_xml = NULL;
	child_rsc->fns->free(child_rsc);
	}

	g_list_free(rsc->children);

	if (clone_data) {
	CRM_ASSERT(clone_data->demote_notify == NULL);
	CRM_ASSERT(clone_data->stop_notify == NULL);
	CRM_ASSERT(clone_data->start_notify == NULL);
	CRM_ASSERT(clone_data->promote_notify == NULL);
	}

	common_free(rsc);
	}

	enum rsc_role_e
	clone_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e clone_role = RSC_ROLE_UNKNOWN;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, current);

	if (a_role > clone_role) {
	clone_role = a_role;
	}
	}

	pe_rsc_trace(rsc, "%s role: %s", rsc->id, role2text(clone_role));
	return clone_role;
	}

	/*!
	* \internal
	* \brief Check whether a clone has an instance for every node
	*
	* \param[in] rsc Clone to check
	* \param[in] data_set Cluster state
	*/
	bool
	pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set)
	{
	if (pe_rsc_is_clone(rsc)) {
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);
	if (clone_data->clone_max == g_list_length(data_set->nodes)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gboolean
	pe__clone_is_filtered(pe_resource_t rsc, GList only_rsc, gboolean check_parent)
	{
	gboolean passes = FALSE;
	clone_variant_data_t *clone_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_clone_variant_data(clone_data, rsc);
	passes = pcmk__str_in_list(ID(clone_data->xml_obj_child), only_rsc, pcmk__str_star_matches);

	if (!passes) {
	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}
	}

	return !passes;
	}

	const char *
	pe__clone_child_id(pe_resource_t *rsc)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);
	return ID(clone_data->xml_obj_child);
	}

	/*!
	* \internal
	* \brief Check whether a clone is ordered
	*
	* \param[in] clone Clone resource to check
	*
	* \return true if clone is ordered, otherwise false
	*/
	bool
	pe__clone_is_ordered(pe_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	return pcmk_is_set(clone_data->flags, pe__clone_ordered);
	}

	/*!
	* \internal
	* \brief Set a clone flag
	*
	* \param[in] clone Clone resource to set flag for
	* \param[in] flag Clone flag to set
	*
	* \return Standard Pacemaker return code (either pcmk_rc_ok if flag was not
	* already set or pcmk_rc_already if it was)
	*/
	int
	pe__set_clone_flag(pe_resource_t *clone, enum pe__clone_flags flag)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	if (pcmk_is_set(clone_data->flags, flag)) {
	return pcmk_rc_already;
	}
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", clone->id,
	clone_data->flags, flag, "flag");
	return pcmk_rc_ok;
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jul 10, 2:34 AM (1 d, 20 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2009746
Default Alt Text: (188 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions