No OneTemporary
Actions

Size

159 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/crm/common/Makefile.am b/include/crm/common/Makefile.am
	index 7d417e498c..c15dfaf385 100644
	--- a/include/crm/common/Makefile.am
	+++ b/include/crm/common/Makefile.am
	@@ -1,55 +1,57 @@
	#
	-# Copyright 2004-2022 the Pacemaker project contributors
	+# Copyright 2004-2023 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	MAINTAINERCLEANFILES = Makefile.in

	headerdir=$(pkgincludedir)/crm/common

	header_HEADERS = acl.h \
	agents.h \
	agents_compat.h \
	cib.h \
	ipc.h \
	ipc_attrd_internal.h \
	ipc_controld.h \
	ipc_pacemakerd.h \
	ipc_schedulerd.h \
	iso8601.h \
	logging.h \
	logging_compat.h \
	mainloop.h \
	mainloop_compat.h \
	nvpair.h \
	output.h \
	+ resources.h \
	results.h \
	results_compat.h \
	+ scheduler.h \
	util.h \
	util_compat.h \
	xml.h \
	xml_compat.h

	noinst_HEADERS = acl_internal.h \
	alerts_internal.h \
	attrd_internal.h \
	cmdline_internal.h \
	health_internal.h \
	internal.h \
	io_internal.h \
	ipc_internal.h \
	iso8601_internal.h \
	lists_internal.h \
	logging_internal.h \
	messages_internal.h \
	options_internal.h \
	output_internal.h \
	remote_internal.h \
	results_internal.h \
	strings_internal.h \
	unittest_internal.h \
	xml_internal.h
	diff --git a/include/crm/common/resources.h b/include/crm/common/resources.h
	new file mode 100644
	index 0000000000..0310fe2515
	--- /dev/null
	+++ b/include/crm/common/resources.h
	@@ -0,0 +1,49 @@
	+/*
	+ * Copyright 2004-2023 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU Lesser General Public License
	+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#ifndef PCMK__CRM_COMMON_RESOURCES__H
	+# define PCMK__CRM_COMMON_RESOURCES__H
	+
	+#ifdef __cplusplus
	+extern "C" {
	+#endif
	+
	+/*!
	+ * \file
	+ * \brief Scheduler API for resources
	+ * \ingroup core
	+ */
	+
	+//! How to recover a resource that is incorrectly active on multiple nodes
	+enum rsc_recovery_type {
	+ pcmk_multiply_active_restart = 0, //!< Stop on all, start on desired
	+ pcmk_multiply_active_stop = 1, //!< Stop on all and leave stopped
	+ pcmk_multiply_active_block = 2, //!< Do nothing to resource
	+ pcmk_multiply_active_unexpected = 3, //!< Stop unexpected instances
	+
	+#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	+ //! \deprecated Use pcmk_multiply_active_restart instead
	+ recovery_stop_start = pcmk_multiply_active_restart,
	+
	+ //! \deprecated Use pcmk_multiply_active_stop instead
	+ recovery_stop_only = pcmk_multiply_active_stop,
	+
	+ //! \deprecated Use pcmk_multiply_active_block instead
	+ recovery_block = pcmk_multiply_active_block,
	+
	+ //! \deprecated Use pcmk_multiply_active_unexpected instead
	+ recovery_stop_unexpected = pcmk_multiply_active_unexpected,
	+#endif
	+};
	+
	+#ifdef __cplusplus
	+}
	+#endif
	+
	+#endif // PCMK__CRM_COMMON_RESOURCES__H
	diff --git a/include/crm/common/scheduler.h b/include/crm/common/scheduler.h
	new file mode 100644
	index 0000000000..9c384ba1b2
	--- /dev/null
	+++ b/include/crm/common/scheduler.h
	@@ -0,0 +1,29 @@
	+/*
	+ * Copyright 2004-2023 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU Lesser General Public License
	+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#ifndef PCMK__CRM_COMMON_SCHEDULER__H
	+# define PCMK__CRM_COMMON_SCHEDULER__H
	+
	+#include <crm/common/resources.h>
	+
	+#ifdef __cplusplus
	+extern "C" {
	+#endif
	+
	+/*!
	+ * \file
	+ * \brief Scheduler API
	+ * \ingroup core
	+ */
	+
	+#ifdef __cplusplus
	+}
	+#endif
	+
	+#endif // PCMK__CRM_COMMON_SCHEDULER__H
	diff --git a/include/crm/pengine/common.h b/include/crm/pengine/common.h
	index 9fe05bd8e5..c461eab044 100644
	--- a/include/crm/pengine/common.h
	+++ b/include/crm/pengine/common.h
	@@ -1,209 +1,203 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_COMMON__H
	# define PCMK__CRM_PENGINE_COMMON__H

	# include <glib.h>
	# include <regex.h>
	# include <crm/common/iso8601.h>
	+# include <crm/common/scheduler.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	extern gboolean was_processing_error;
	extern gboolean was_processing_warning;

	/* The order is (partially) significant here; the values from action_fail_ignore
	* through action_fail_fence are in order of increasing severity.
	*
	* @COMPAT The values should be ordered and numbered per the "TODO" comments
	* below, so all values are in order of severity and there is room for
	* future additions, but that would break API compatibility.
	* @TODO For now, we just use a function to compare the values specially, but
	* at the next compatibility break, we should arrange things properly.
	*/
	enum action_fail_response {
	action_fail_ignore, // @TODO = 10
	// @TODO action_fail_demote = 20,
	action_fail_recover, // @TODO = 30
	// @TODO action_fail_reset_remote = 40,
	// @TODO action_fail_restart_container = 50,
	action_fail_migrate, // @TODO = 60
	action_fail_block, // @TODO = 70
	action_fail_stop, // @TODO = 80
	action_fail_standby, // @TODO = 90
	action_fail_fence, // @TODO = 100

	// @COMPAT Values below here are out of order for API compatibility

	action_fail_restart_container,

	/* This is reserved for internal use for remote node connection resources.
	* Fence the remote node if stonith is enabled, otherwise attempt to recover
	* the connection resource. This allows us to specify types of connection
	* resource failures that should result in fencing the remote node
	* (for example, recurring monitor failures).
	*/
	action_fail_reset_remote,

	action_fail_demote,
	};

	/* the "done" action must be the "pre" action +1 */
	enum action_tasks {
	no_action,
	monitor_rsc,
	stop_rsc,
	stopped_rsc,
	start_rsc,
	started_rsc,
	action_notify,
	action_notified,
	action_promote,
	action_promoted,
	action_demote,
	action_demoted,
	shutdown_crm,
	stonith_node
	};

	-enum rsc_recovery_type {
	- recovery_stop_start,
	- recovery_stop_only,
	- recovery_block,
	- recovery_stop_unexpected,
	-};
	-
	enum rsc_start_requirement {
	rsc_req_nothing, /* Allowed by custom_action() */
	rsc_req_quorum, /* Enforced by custom_action() */
	rsc_req_stonith /* Enforced by native_start_constraints() */
	};

	//! Possible roles that a resource can be in
	enum rsc_role_e {
	RSC_ROLE_UNKNOWN = 0,
	RSC_ROLE_STOPPED = 1,
	RSC_ROLE_STARTED = 2,
	RSC_ROLE_UNPROMOTED = 3,
	RSC_ROLE_PROMOTED = 4,

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	//! \deprecated Use RSC_ROLE_UNPROMOTED instead
	RSC_ROLE_SLAVE = RSC_ROLE_UNPROMOTED,

	//! \deprecated Use RSC_ROLE_PROMOTED instead
	RSC_ROLE_MASTER = RSC_ROLE_PROMOTED,
	#endif
	};

	# define RSC_ROLE_MAX (RSC_ROLE_PROMOTED + 1)

	# define RSC_ROLE_UNKNOWN_S "Unknown"
	# define RSC_ROLE_STOPPED_S "Stopped"
	# define RSC_ROLE_STARTED_S "Started"
	# define RSC_ROLE_UNPROMOTED_S "Unpromoted"
	# define RSC_ROLE_PROMOTED_S "Promoted"
	# define RSC_ROLE_UNPROMOTED_LEGACY_S "Slave"
	# define RSC_ROLE_PROMOTED_LEGACY_S "Master"

	//! Deprecated
	enum pe_print_options {
	pe_print_log = (1 << 0),
	pe_print_html = (1 << 1),
	pe_print_ncurses = (1 << 2),
	pe_print_printf = (1 << 3),
	pe_print_dev = (1 << 4), //! Ignored
	pe_print_details = (1 << 5), //! Ignored
	pe_print_max_details = (1 << 6), //! Ignored
	pe_print_rsconly = (1 << 7),
	pe_print_ops = (1 << 8),
	pe_print_suppres_nl = (1 << 9),
	pe_print_xml = (1 << 10),
	pe_print_brief = (1 << 11),
	pe_print_pending = (1 << 12),
	pe_print_clone_details = (1 << 13),
	pe_print_clone_active = (1 << 14), // Print clone instances only if active
	pe_print_implicit = (1 << 15) // Print implicitly created resources
	};

	const char *task2text(enum action_tasks task);
	enum action_tasks text2task(const char *task);
	enum rsc_role_e text2role(const char *role);
	const char *role2text(enum rsc_role_e role);
	const char *fail2text(enum action_fail_response fail);

	const char pe_pref(GHashTable options, const char *name);

	/*!
	* \brief Get readable description of a recovery type
	*
	* \param[in] type Recovery type
	*
	* \return Static string describing \p type
	*/
	static inline const char *
	recovery2text(enum rsc_recovery_type type)
	{
	switch (type) {
	- case recovery_stop_only:
	+ case pcmk_multiply_active_stop:
	return "shutting it down";
	- case recovery_stop_start:
	+ case pcmk_multiply_active_restart:
	return "attempting recovery";
	- case recovery_block:
	+ case pcmk_multiply_active_block:
	return "waiting for an administrator";
	- case recovery_stop_unexpected:
	+ case pcmk_multiply_active_unexpected:
	return "stopping unexpected instances";
	}
	return "Unknown";
	}

	typedef struct pe_re_match_data {
	char *string;
	int nregs;
	regmatch_t *pmatch;
	} pe_re_match_data_t;

	typedef struct pe_match_data {
	pe_re_match_data_t *re;
	GHashTable *params;
	GHashTable *meta;
	} pe_match_data_t;

	typedef struct pe_rsc_eval_data {
	const char *standard;
	const char *provider;
	const char *agent;
	} pe_rsc_eval_data_t;

	typedef struct pe_op_eval_data {
	const char *op_name;
	guint interval;
	} pe_op_eval_data_t;

	typedef struct pe_rule_eval_data {
	GHashTable *node_hash; // Only used with g_hash_table_lookup()
	enum rsc_role_e role;
	crm_time_t *now; // @COMPAT could be const
	pe_match_data_t *match_data; // @COMPAT could be const
	pe_rsc_eval_data_t *rsc_data; // @COMPAT could be const
	pe_op_eval_data_t *op_data; // @COMPAT could be const
	} pe_rule_eval_data_t;

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/pengine/common_compat.h>
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
	index 10ed831cbc..010828f740 100644
	--- a/lib/pacemaker/pcmk_sched_primitive.c
	+++ b/lib/pacemaker/pcmk_sched_primitive.c
	@@ -1,1631 +1,1631 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdint.h> // uint8_t, uint32_t

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static void stop_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void start_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void demote_resource(pe_resource_t rsc, pe_node_t node, bool optional);
	static void promote_resource(pe_resource_t rsc, pe_node_t node,
	bool optional);
	static void assert_role_error(pe_resource_t rsc, pe_node_t node,
	bool optional);

	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the immediate next role when transitioning from one role
	* to a target role. For example, when going from Stopped to Promoted, the
	* next role is Unpromoted, because the resource must be started before it
	* can be promoted. The current state then becomes Started, which is fed
	* into this array again, giving a next role of Promoted.
	*
	* Current role Immediate next role Final target role
	* ------------ ------------------- -----------------
	*/
	/* Unknown / { RSC_ROLE_UNKNOWN, / Unknown */
	RSC_ROLE_STOPPED, /* Stopped */
	RSC_ROLE_STOPPED, /* Started */
	RSC_ROLE_STOPPED, /* Unpromoted */
	RSC_ROLE_STOPPED, /* Promoted */
	},
	/* Stopped / { RSC_ROLE_STOPPED, / Unknown */
	RSC_ROLE_STOPPED, /* Stopped */
	RSC_ROLE_STARTED, /* Started */
	RSC_ROLE_UNPROMOTED, /* Unpromoted */
	RSC_ROLE_UNPROMOTED, /* Promoted */
	},
	/* Started / { RSC_ROLE_STOPPED, / Unknown */
	RSC_ROLE_STOPPED, /* Stopped */
	RSC_ROLE_STARTED, /* Started */
	RSC_ROLE_UNPROMOTED, /* Unpromoted */
	RSC_ROLE_PROMOTED, /* Promoted */
	},
	/* Unpromoted / { RSC_ROLE_STOPPED, / Unknown */
	RSC_ROLE_STOPPED, /* Stopped */
	RSC_ROLE_STOPPED, /* Started */
	RSC_ROLE_UNPROMOTED, /* Unpromoted */
	RSC_ROLE_PROMOTED, /* Promoted */
	},
	/* Promoted / { RSC_ROLE_STOPPED, / Unknown */
	RSC_ROLE_UNPROMOTED, /* Stopped */
	RSC_ROLE_UNPROMOTED, /* Started */
	RSC_ROLE_UNPROMOTED, /* Unpromoted */
	RSC_ROLE_PROMOTED, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Function to schedule actions needed for a role change
	*
	* \param[in,out] rsc Resource whose role is changing
	* \param[in,out] node Node where resource will be in its next role
	* \param[in] optional Whether scheduled actions should be optional
	*/
	typedef void (rsc_transition_fn)(pe_resource_t rsc, pe_node_t *node,
	bool optional);

	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the function needed to transition directly from one role
	* to another. NULL indicates that nothing is needed.
	*
	* Current role Transition function Next role
	* ------------ ------------------- ----------
	*/
	/* Unknown / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	assert_role_error, /* Started */
	assert_role_error, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Stopped / { assert_role_error, / Unknown */
	NULL, /* Stopped */
	start_resource, /* Started */
	start_resource, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Started / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	NULL, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Unpromoted / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	stop_resource, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Promoted / { assert_role_error, / Unknown */
	demote_resource, /* Stopped */
	demote_resource, /* Started */
	demote_resource, /* Unpromoted */
	NULL, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Get a list of a resource's allowed nodes sorted by node score
	*
	* \param[in] rsc Resource to check
	*
	* \return List of allowed nodes sorted by node score
	*/
	static GList *
	sorted_allowed_nodes(const pe_resource_t *rsc)
	{
	if (rsc->allowed_nodes != NULL) {
	GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);

	if (nodes != NULL) {
	return pcmk__sort_nodes(nodes, pe__current_node(rsc));
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Assign a resource to its best allowed node, if possible
	*
	* \param[in,out] rsc Resource to choose a node for
	* \param[in] prefer If not \c NULL, prefer this node when all else
	* equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return true if \p rsc could be assigned to a node, otherwise false
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	static bool
	assign_best_node(pe_resource_t rsc, const pe_node_t prefer, bool stop_if_fail)
	{
	GList *nodes = NULL;
	pe_node_t *chosen = NULL;
	pe_node_t *best = NULL;
	const pe_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);

	if (prefer == NULL) {
	prefer = most_free_node;
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	// We've already finished assignment of resources to nodes
	return rsc->allocated_to != NULL;
	}

	// Sort allowed nodes by score
	nodes = sorted_allowed_nodes(rsc);
	if (nodes != NULL) {
	best = (pe_node_t *) nodes->data; // First node has best score
	}

	if ((prefer != NULL) && (nodes != NULL)) {
	// Get the allowed node version of prefer
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
	pe__node_name(prefer), rsc->id);

	/* Favor the preferred node as long as its score is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's score is less than INFINITY.
	*/
	} else if (chosen->weight < best->weight) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen, true, false)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else {
	pe_rsc_trace(rsc,
	"Chose preferred node %s for %s "
	"(ignoring %d candidates)",
	pe__node_name(chosen), rsc->id, g_list_length(nodes));
	}
	}

	if ((chosen == NULL) && (best != NULL)) {
	/* Either there is no preferred node, or the preferred node is not
	* suitable, but another node is allowed to run the resource.
	*/

	chosen = best;

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& (chosen->weight > 0) // Zero not acceptable
	&& pcmk__node_available(chosen, false, false)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* pcmk__assign_instances() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unassigned instances to prefer a node that's already
	* running another instance.
	*/
	pe_node_t *running = pe__current_node(rsc);

	if (running == NULL) {
	// Nothing to do

	} else if (!pcmk__node_available(running, true, false)) {
	pe_rsc_trace(rsc,
	"Current node for %s (%s) can't run resources",
	rsc->id, pe__node_name(running));

	} else {
	int nodes_with_best_score = 1;

	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pe_node_t allowed = (pe_node_t ) iter->data;

	if (allowed->weight != chosen->weight) {
	// The nodes are sorted by score, so no more are equal
	break;
	}
	if (pe__same_node(allowed, running)) {
	// Scores are equal, so prefer the current node
	chosen = allowed;
	}
	nodes_with_best_score++;
	}

	if (nodes_with_best_score > 1) {
	uint8_t log_level = LOG_INFO;

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}
	do_crm_log(log_level,
	"Chose %s for %s from %d nodes with score %s",
	pe__node_name(chosen), rsc->id,
	nodes_with_best_score,
	pcmk_readable_score(chosen->weight));
	}
	}
	}

	pe_rsc_trace(rsc, "Chose %s for %s from %d candidates",
	pe__node_name(chosen), rsc->id, g_list_length(nodes));
	}

	pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
	g_list_free(nodes);
	return rsc->allocated_to != NULL;
	}

	/*!
	* \internal
	* \brief Apply a "this with" colocation to a node's allowed node scores
	*
	* \param[in,out] colocation Colocation to apply
	* \param[in,out] rsc Resource being assigned
	*/
	static void
	apply_this_with(pcmk__colocation_t colocation, pe_resource_t rsc)
	{
	GHashTable *archive = NULL;
	pe_resource_t *other = colocation->primary;

	// In certain cases, we will need to revert the node scores
	if ((colocation->dependent_role >= RSC_ROLE_PROMOTED)
	\|\| ((colocation->score < 0) && (colocation->score > -INFINITY))) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	if (pcmk_is_set(other->flags, pe_rsc_provisional)) {
	pe_rsc_trace(rsc,
	"%s: Assigning colocation %s primary %s first"
	"(score=%d role=%s)",
	rsc->id, colocation->id, other->id,
	colocation->score, role2text(colocation->dependent_role));
	other->cmds->assign(other, NULL, true);
	}

	// Apply the colocation score to this resource's allowed node scores
	rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
	if ((archive != NULL)
	&& !pcmk__any_node_available(rsc->allowed_nodes)) {
	pe_rsc_info(rsc,
	"%s: Reverting scores from colocation with %s "
	"because no nodes allowed",
	rsc->id, other->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive != NULL) {
	g_hash_table_destroy(archive);
	}
	}

	/*!
	* \internal
	* \brief Update a Pacemaker Remote node once its connection has been assigned
	*
	* \param[in] connection Connection resource that has been assigned
	*/
	static void
	remote_connection_assigned(const pe_resource_t *connection)
	{
	pe_node_t *remote_node = pe_find_node(connection->cluster->nodes,
	connection->id);

	CRM_CHECK(remote_node != NULL, return);

	if ((connection->allocated_to != NULL)
	&& (connection->next_role != RSC_ROLE_STOPPED)) {

	crm_trace("Pacemaker Remote node %s will be online",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	if (remote_node->details->unseen) {
	// Avoid unnecessary fence, since we will attempt connection
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Pacemaker Remote node %s will be shut down "
	"(%sassigned connection's next role is %s)",
	remote_node->details->id,
	((connection->allocated_to == NULL)? "un" : ""),
	role2text(connection->next_role));
	remote_node->details->shutdown = TRUE;
	}
	}

	/*!
	* \internal
	* \brief Assign a primitive resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pe_node_t *
	pcmk__primitive_assign(pe_resource_t rsc, const pe_node_t prefer,
	bool stop_if_fail)
	{
	GList *this_with_colocations = NULL;
	GList *with_this_colocations = NULL;
	GList *iter = NULL;
	pcmk__colocation_t *colocation = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native));

	// Never assign a child without parent being assigned first
	if ((rsc->parent != NULL)
	&& !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "%s: Assigning parent %s first",
	rsc->id, rsc->parent->id);
	rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	// Assignment has already been done
	const char *node_name = "no node";

	if (rsc->allocated_to != NULL) {
	node_name = pe__node_name(rsc->allocated_to);
	}
	pe_rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
	return rsc->allocated_to;
	}

	// Ensure we detect assignment loops
	if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pe__set_resource_flags(rsc, pe_rsc_allocating);

	pe__show_node_scores(true, rsc, "Pre-assignment", rsc->allowed_nodes,
	rsc->cluster);

	this_with_colocations = pcmk__this_with_colocations(rsc);
	with_this_colocations = pcmk__with_this_colocations(rsc);

	// Apply mandatory colocations first, to satisfy as many as possible
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	pe__show_node_scores(true, rsc, "Mandatory-colocations",
	rsc->allowed_nodes, rsc->cluster);

	// Then apply optional colocations
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	g_list_free(this_with_colocations);
	g_list_free(with_this_colocations);

	if (rsc->next_role == RSC_ROLE_STOPPED) {
	pe_rsc_trace(rsc,
	"Banning %s from all nodes because it will be stopped",
	rsc->id);
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE,
	rsc->cluster);

	} else if ((rsc->next_role > rsc->role)
	&& !pcmk_is_set(rsc->cluster->flags, pe_flag_have_quorum)
	&& (rsc->cluster->no_quorum_policy == no_quorum_freeze)) {
	crm_notice("Resource %s cannot be elevated from %s to %s due to "
	"no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
	}

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags, pe_flag_show_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	// Unmanage resource if fencing is enabled but no device is configured
	if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)
	&& !pcmk_is_set(rsc->cluster->flags, pe_flag_have_stonith_resource)) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	// Unmanaged resources stay on their current node
	const char *reason = NULL;
	pe_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_PROMOTED) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"), reason);
	pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);

	} else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stop_everything)) {
	// Must stop at some point, but be consistent with stop_if_fail
	if (stop_if_fail) {
	pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources",
	rsc->id);
	}
	pcmk__assign_resource(rsc, NULL, true, stop_if_fail);

	} else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
	// Assignment failed
	if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if ((rsc->running_on != NULL) && stop_if_fail) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}
	}

	pe__clear_resource_flags(rsc, pe_rsc_allocating);

	if (rsc->is_remote_node) {
	remote_connection_assigned(rsc);
	}

	return rsc->allocated_to;
	}

	/*!
	* \internal
	* \brief Schedule actions to bring resource down and back to current role
	*
	* \param[in,out] rsc Resource to restart
	* \param[in,out] current Node that resource should be brought down on
	* \param[in] need_stop Whether the resource must be stopped
	* \param[in] need_promote Whether the resource must be promoted
	*
	* \return Role that resource would have after scheduled actions are taken
	*/
	static void
	schedule_restart_actions(pe_resource_t rsc, pe_node_t current,
	bool need_stop, bool need_promote)
	{
	enum rsc_role_e role = rsc->role;
	enum rsc_role_e next_role;
	rsc_transition_fn fn = NULL;

	pe__set_resource_flags(rsc, pe_rsc_restarting);

	// Bring resource down to a stop on its current node
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, current, !need_stop);
	role = next_role;
	}

	// Bring resource up to its next role on its next node
	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pe_rsc_block)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == RSC_ROLE_PROMOTED) && need_promote) {
	required = true;
	}
	pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, !required);
	role = next_role;
	}

	pe__clear_resource_flags(rsc, pe_rsc_restarting);
	}

	/*!
	* \internal
	* \brief If a resource's next role is not explicitly specified, set a default
	*
	* \param[in,out] rsc Resource to set next role for
	*
	* \return "explicit" if next role was explicitly set, otherwise "implicit"
	*/
	static const char *
	set_default_next_role(pe_resource_t *rsc)
	{
	if (rsc->next_role != RSC_ROLE_UNKNOWN) {
	return "explicit";
	}

	if (rsc->allocated_to == NULL) {
	pe__set_next_role(rsc, RSC_ROLE_STOPPED, "assignment");
	} else {
	pe__set_next_role(rsc, RSC_ROLE_STARTED, "assignment");
	}
	return "implicit";
	}

	/*!
	* \internal
	* \brief Create an action to represent an already pending start
	*
	* \param[in,out] rsc Resource to create start action for
	*/
	static void
	create_pending_start(pe_resource_t *rsc)
	{
	pe_action_t *start = NULL;

	pe_rsc_trace(rsc,
	"Creating action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	pe__set_action_flags(start, pe_action_print_always);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to take a resource to its next role
	*
	* \param[in,out] rsc Resource to schedule actions for
	*/
	static void
	schedule_role_transition_actions(pe_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	while (role != rsc->next_role) {
	enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
	rsc_transition_fn fn = NULL;

	pe_rsc_trace(rsc,
	"Creating action to take %s from %s to %s (ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, false);
	role = next_role;
	}
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given primitive resource
	*
	* \param[in,out] rsc Primitive resource to create actions for
	*/
	void
	pcmk__primitive_create_actions(pe_resource_t *rsc)
	{
	bool need_stop = false;
	bool need_promote = false;
	bool is_moving = false;
	bool allow_migrate = false;
	bool multiply_active = false;

	pe_node_t *current = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	const char *next_role_source = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native));

	next_role_source = set_default_next_role(rsc);
	pe_rsc_trace(rsc,
	"Creating all actions for %s transition from %s to %s "
	"(%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	next_role_source, pe__node_name(rsc->allocated_to));

	current = rsc->fns->active_node(rsc, &num_all_active, &num_clean_active);

	g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
	rsc);

	if ((current != NULL) && (rsc->allocated_to != NULL)
	&& !pe__same_node(current, rsc->allocated_to)
	&& (rsc->next_role >= RSC_ROLE_STARTED)) {

	pe_rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, pe__node_name(current),
	pe__node_name(rsc->allocated_to));
	is_moving = true;
	allow_migrate = pcmk__rsc_can_migrate(rsc, current);

	// This is needed even if migrating (though I'm not sure why ...)
	need_stop = true;
	}

	// Check whether resource is partially migrated and/or multiply active
	if ((rsc->partial_migration_source != NULL)
	&& (rsc->partial_migration_target != NULL)
	&& allow_migrate && (num_all_active == 2)
	&& pe__same_node(current, rsc->partial_migration_source)
	&& pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
	/* A partial migration is in progress, and the migration target remains
	* the same as when the migration began.
	*/
	pe_rsc_trace(rsc, "Partial migration of %s from %s to %s will continue",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));

	} else if ((rsc->partial_migration_source != NULL)
	\|\| (rsc->partial_migration_target != NULL)) {
	// A partial migration is in progress but can't be continued

	if (num_all_active > 2) {
	// The resource is migrating and multiply active!
	crm_notice("Forcing recovery of %s because it is migrating "
	"from %s to %s and possibly active elsewhere",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	} else {
	// The migration source or target isn't available
	crm_notice("Forcing recovery of %s because it can no longer "
	"migrate from %s to %s",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	}
	need_stop = true;
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = false;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	multiply_active = (num_all_active > 1);
	} else {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* stonith-enabled is false). We can start such resources elsewhere
	* before fencing completes, and if we considered the resource active on
	* the failed node, we would attempt recovery for being active on
	* multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	}

	if (multiply_active) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pe_proc_err("%s resource %s might be active on %u nodes (%s)",
	pcmk__s(class, "Untyped"), rsc->id, num_all_active,
	recovery2text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
	"#Resource_is_Too_Active for more information");

	switch (rsc->recovery_type) {
	- case recovery_stop_start:
	+ case pcmk_multiply_active_restart:
	need_stop = true;
	break;
	- case recovery_stop_unexpected:
	+ case pcmk_multiply_active_unexpected:
	need_stop = true; // stop_resource() will skip expected node
	pe__set_resource_flags(rsc, pe_rsc_stop_unexpected);
	break;
	default:
	break;
	}

	} else {
	pe__clear_resource_flags(rsc, pe_rsc_stop_unexpected);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
	create_pending_start(rsc);
	}

	if (is_moving) {
	// Remaining tests are only for resources staying where they are

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
	need_stop = true;
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == RSC_ROLE_PROMOTED) {
	need_promote = true;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = true;

	} else if ((rsc->role > RSC_ROLE_STARTED) && (current != NULL)
	&& (rsc->allocated_to != NULL)) {
	pe_action_t *start = NULL;

	pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	if (!pcmk_is_set(start->flags, pe_action_optional)) {
	// Recovery of a promoted resource
	pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = true;
	}
	}

	// Create any actions needed to bring resource down and back up to same role
	schedule_restart_actions(rsc, current, need_stop, need_promote);

	// Create any actions needed to take resource from this role to the next
	schedule_role_transition_actions(rsc);

	pcmk__create_recurring_actions(rsc);

	if (allow_migrate) {
	pcmk__create_migration_actions(rsc, current);
	}
	}

	/*!
	* \internal
	* \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
	*
	* \param[in] rsc Resource to check
	*/
	static void
	rsc_avoids_remote_nodes(const pe_resource_t *rsc)
	{
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (node->details->remote_rsc != NULL) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(const pe_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
	}

	return allowed_nodes;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a primitive resource
	*
	* \param[in,out] rsc Primitive resource to create implicit constraints for
	*/
	void
	pcmk__primitive_internal_constraints(pe_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;
	bool check_unfencing = false;
	bool check_utilization = false;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native));

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc,
	"Skipping implicit constraints for unmanaged resource %s",
	rsc->id);
	return;
	}

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
	&& pcmk_is_set(rsc->cluster->flags,
	pe_flag_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(rsc->cluster->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_optional\|pe_order_implies_then\|pe_order_restart,
	rsc->cluster);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pe_rsc_promotable)
	\|\| (rsc->role > RSC_ROLE_UNPROMOTED)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_promoted_implies_first, rsc->cluster);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
	rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, rsc->cluster);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
	NULL, pe_order_same_node\|pe_order_then_cancels_first,
	rsc->cluster);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| (rsc->container != NULL)) {
	allowed_nodes = allowed_nodes_as_list(rsc);
	}

	if (check_unfencing) {
	g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->container != NULL) {
	pe_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Guest resources are not allowed to run on Pacemaker Remote nodes,
	* to avoid nesting remotes. However, bundles are allowed.
	*/
	if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->container, RSC_STATUS, rsc,
	RSC_STOP, pe_order_optional);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
	rsc->container);
	}

	if (remote_rsc != NULL) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pe_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	pcmk__new_ordering(rsc->container,
	pcmk__op_key(rsc->container->id, RSC_START, 0),
	NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
	NULL,
	pe_order_implies_then\|pe_order_runnable_left,
	rsc->cluster);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id, RSC_STOP, 0),
	NULL, pe_order_implies_first, rsc->cluster);

	if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL,
	pcmk__coloc_influence);
	}
	}

	if (rsc->is_remote_node \|\| pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	/* Remote connections and fencing devices are not allowed to run on
	* Pacemaker Remote nodes
	*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__primitive_apply_coloc_score(pe_resource_t *dependent,
	const pe_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	// Always process on behalf of primary resource
	primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
	return;
	}

	filter_results = pcmk__colocation_affects(dependent, primary, colocation,
	false);
	pe_rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((colocation->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, colocation->id, colocation->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, colocation);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_scores(dependent, primary, colocation);
	break;
	default: // pcmk__coloc_affects_nothing
	return;
	}
	}

	/* Primitive implementation of
	* resource_alloc_functions_t:with_this_colocations()
	*/
	void
	pcmk__with_primitive_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native) && (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_with_this(list, colocation, orig_rsc);
	}
	}
	}
	}

	/* Primitive implementation of
	* resource_alloc_functions_t:this_with_colocations()
	*/
	void
	pcmk__primitive_with_colocations(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native) && (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given primitive resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node (ignored)
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__primitive_action_flags(pe_action_t action, const pe_node_t node)
	{
	CRM_ASSERT(action != NULL);
	return (uint32_t) action->flags;
	}

	/*!
	* \internal
	* \brief Check whether a node is a multiply active resource's expected node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p rsc is multiply active with multiple-active set to
	* stop_unexpected, and \p node is the node where it will remain active
	* \note This assumes that the resource's next role cannot be changed to stopped
	* after this is called, which should be reasonable if status has already
	* been unpacked and resources have been assigned to nodes.
	*/
	static bool
	is_expected_node(const pe_resource_t rsc, const pe_node_t node)
	{
	return pcmk_all_flags_set(rsc->flags,
	pe_rsc_stop_unexpected\|pe_rsc_restarting)
	&& (rsc->next_role > RSC_ROLE_STOPPED)
	&& pe__same_node(rsc->allocated_to, node);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to stop a resource wherever it is active
	*
	* \param[in,out] rsc Resource being stopped
	* \param[in] node Node where resource is being stopped (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	stop_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t current = (pe_node_t ) iter->data;
	pe_action_t *stop = NULL;

	if (is_expected_node(rsc, current)) {
	/* We are scheduling restart actions for a multiply active resource
	* with multiple-active=stop_unexpected, and this is where it should
	* not be stopped.
	*/
	pe_rsc_trace(rsc,
	"Skipping stop of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	continue;
	}

	if (rsc->partial_migration_target != NULL) {
	// Continue migration if node originally was and remains target
	if (pe__same_node(current, rsc->partial_migration_target)
	&& pe__same_node(current, rsc->allocated_to)) {
	pe_rsc_trace(rsc,
	"Skipping stop of %s on %s "
	"because partial migration there will continue",
	rsc->id, pe__node_name(current));
	continue;
	} else {
	pe_rsc_trace(rsc,
	"Forcing stop of %s on %s "
	"because migration target changed",
	rsc->id, pe__node_name(current));
	optional = false;
	}
	}

	pe_rsc_trace(rsc, "Scheduling stop of %s on %s",
	rsc->id, pe__node_name(current));
	stop = stop_action(rsc, current, optional);

	if (rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", true);
	} else if (pcmk_all_flags_set(rsc->flags, pe_rsc_restarting
	\|pe_rsc_stop_unexpected)) {
	/* We are stopping a multiply active resource on a node that is
	* not its expected node, and we are still scheduling restart
	* actions, so the stop is for being multiply active.
	*/
	pe_action_set_reason(stop, "being multiply active", true);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe__clear_action_flags(stop, pe_action_runnable);
	}

	if (pcmk_is_set(rsc->cluster->flags, pe_flag_remove_after_stop)) {
	pcmk__schedule_cleanup(rsc, current, optional);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
	pe_action_t *unfence = pe_fence_op(current, "on", true, NULL, false,
	rsc->cluster);

	order_actions(stop, unfence, pe_order_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pe_proc_err("Stopping %s until %s can be unfenced",
	rsc->id, pe__node_name(current));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to start a resource on a node
	*
	* \param[in,out] rsc Resource being started
	* \param[in,out] node Node where resource should be started
	* \param[in] optional Whether actions should be optional
	*/
	static void
	start_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	pe_action_t *start = NULL;

	CRM_ASSERT(node != NULL);

	pe_rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node), node->weight);
	start = start_action(rsc, node, TRUE);

	pcmk__order_vs_unfence(rsc, node, start, pe_order_implies_then);

	if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
	pe__clear_action_flags(start, pe_action_optional);
	}

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the start becomes necessary for other
	* reasons later.
	*/
	pe_rsc_trace(rsc,
	"Start of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pe__set_action_flags(start, pe_action_pseudo);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to promote a resource on a node
	*
	* \param[in,out] rsc Resource being promoted
	* \param[in] node Node where resource should be promoted
	* \param[in] optional Whether actions should be optional
	*/
	static void
	promote_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool runnable = true;

	CRM_ASSERT(node != NULL);

	// Any start must be runnable for promotion to be runnable
	action_list = pe__resource_actions(rsc, node, RSC_START, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t start = (pe_action_t ) iter->data;

	if (!pcmk_is_set(start->flags, pe_action_runnable)) {
	runnable = false;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	pe_action_t *promote = promote_action(rsc, node, optional);

	pe_rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node));

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the promote becomes necessary for
	* other reasons later.
	*/
	pe_rsc_trace(rsc,
	"Promotion of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pe__set_action_flags(promote, pe_action_pseudo);
	}
	} else {
	pe_rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
	rsc->id, pe__node_name(node));
	action_list = pe__resource_actions(rsc, node, RSC_PROMOTE, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pe_action_t promote = (pe_action_t ) iter->data;

	pe__clear_action_flags(promote, pe_action_runnable);
	}
	g_list_free(action_list);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to demote a resource wherever it is active
	*
	* \param[in,out] rsc Resource being demoted
	* \param[in] node Node where resource should be demoted (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	demote_resource(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	/* Since this will only be called for a primitive (possibly as an instance
	* of a collective resource), the resource is multiply active if it is
	* running on more than one node, so we want to demote on all of them as
	* part of recovery, regardless of which one is the desired node.
	*/
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t current = (pe_node_t ) iter->data;

	if (is_expected_node(rsc, current)) {
	pe_rsc_trace(rsc,
	"Skipping demote of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	} else {
	pe_rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(current));
	demote_action(rsc, current, optional);
	}
	}
	}

	static void
	assert_role_error(pe_resource_t rsc, pe_node_t node, bool optional)
	{
	CRM_ASSERT(false);
	}

	/*!
	* \internal
	* \brief Schedule cleanup of a resource
	*
	* \param[in,out] rsc Resource to clean up
	* \param[in] node Node to clean up on
	* \param[in] optional Whether clean-up should be optional
	*/
	void
	pcmk__schedule_cleanup(pe_resource_t rsc, const pe_node_t node, bool optional)
	{
	/* If the cleanup is required, its orderings are optional, because they're
	* relevant only if both actions are required. Conversely, if the cleanup is
	* optional, the orderings make the then action required if the first action
	* becomes required.
	*/
	uint32_t flag = optional? pe_order_implies_then : pe_order_optional;

	CRM_CHECK((rsc != NULL) && (node != NULL), return);

	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
	rsc->id, pe__node_name(node));
	return;
	}

	if (node->details->unclean \|\| !node->details->online) {
	pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
	rsc->id, pe__node_name(node));
	return;
	}

	crm_notice("Scheduling clean-up of %s on %s", rsc->id, pe__node_name(node));
	delete_action(rsc, node, optional);

	// stop -> clean-up -> start
	pcmk__order_resource_actions(rsc, RSC_STOP, rsc, RSC_DELETE, flag);
	pcmk__order_resource_actions(rsc, RSC_DELETE, rsc, RSC_START, flag);
	}

	/*!
	* \internal
	* \brief Add primitive meta-attributes relevant to graph actions to XML
	*
	* \param[in] rsc Primitive resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__primitive_add_graph_meta(const pe_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;
	char *value = NULL;
	const pe_resource_t *parent = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native) && (xml != NULL));

	/* Clone instance numbers get set internally as meta-attributes, and are
	* needed in the transition graph (for example, to tell unique clone
	* instances apart).
	*/
	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	if (value != NULL) {
	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	// Not sure if this one is really needed ...
	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value != NULL) {
	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	/* The container meta-attribute can be set on the primitive itself or one of
	* its parents (for example, a group inside a container resource), so check
	* them all, and keep the highest one found.
	*/
	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container != NULL) {
	crm_xml_add(xml, CRM_META "_" XML_RSC_ATTR_CONTAINER,
	parent->container->id);
	}
	}

	/* Bundle replica children will get their external-ip set internally as a
	* meta-attribute. The graph action needs it, but under a different naming
	* convention than other meta-attributes.
	*/
	value = g_hash_table_lookup(rsc->meta, "external-ip");
	if (value != NULL) {
	crm_xml_add(xml, "pcmk_external_ip", value);
	}
	}

	// Primitive implementation of resource_alloc_functions_t:add_utilization()
	void
	pcmk__primitive_add_utilization(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}

	/*!
	* \internal
	* \brief Get epoch time of node's shutdown attribute (or now if none)
	*
	* \param[in,out] node Node to check
	*
	* \return Epoch time corresponding to shutdown attribute if set or now if not
	*/
	static time_t
	shutdown_time(pe_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown != NULL) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return (result == 0)? get_effective_time(node->details->data_set) : result;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it's not locked to the node
	*
	* \param[in] data Node to check
	* \param[in,out] user_data Resource to check
	*/
	static void
	ban_if_not_locked(gpointer data, gpointer user_data)
	{
	const pe_node_t node = (const pe_node_t ) data;
	pe_resource_t rsc = (pe_resource_t ) user_data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, rsc->cluster);
	}
	}

	// Primitive implementation of resource_alloc_functions_t:shutdown_lock()
	void
	pcmk__primitive_shutdown_lock(pe_resource_t *rsc)
	{
	const char *class = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native));

	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	// Fence devices and remote connections can't be locked
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc, rsc->cluster)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node, rsc->cluster);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, pe__node_name(node));
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (rsc->cluster->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, pe__node_name(rsc->lock_node),
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, rsc->cluster);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, pe__node_name(rsc->lock_node));
	}

	// If resource is locked to one node, ban it from all other nodes
	g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 025b6b2a14..3ef6a8ddb6 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1178 +1,1178 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pe_node_t active_node(const pe_resource_t rsc, unsigned int *count_all,
	unsigned int *count_clean);

	resource_object_functions_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pe_native;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pe_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pe_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pe_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pe_container;
	}

	return pe_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pe_resource_t * rsc, pe_rule_eval_data_t * rule_data, GHashTable * meta_hash, pe_working_set_t * data_set)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pe_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS,
	rule_data, parent_orig_meta, NULL, FALSE, data_set);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
	pe_node_t * node, pe_working_set_t * data_set)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, data_set);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pe_resource_t rsc,
	const pe_node_t node, pe_working_set_t data_set)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = data_set->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, data_set);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, data_set);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE, data_set);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, RSC_ROLE_STARTED_S, RSC_ROLE_UNPROMOTED_S,
	RSC_ROLE_UNPROMOTED_LEGACY_S, NULL)) {
	role = RSC_ROLE_UNKNOWN_S;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *clone = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", crm_element_name(xml_obj));
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pe_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	XML_ATTR_ID, template_ref);
	if (template == NULL) {
	pe_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_replace(new_xml, XML_ATTR_ID, id);

	clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
	if(clone) {
	crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
	}

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	/* Disable multi-level templates for now */
	/*if(unpack_template(new_xml, expanded_xml, data_set) == FALSE) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;

	return FALSE;
	} */

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode * xml_obj, pe_working_set_t * data_set)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pe_err("No resource object for processing resource list of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pe_err("'%s' object must have a id", crm_element_name(xml_obj));
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pe_err("The resource object '%s' should not reference itself", id);
	return FALSE;
	}

	if (add_tag_ref(data_set->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pe_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__str_eq(crm_element_name(rsc->xml), PCMK_XE_PROMOTABLE_LEGACY,
	pcmk__str_casei)) {
	/* @TODO in some future version, pe_warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] data_set Cluster working set
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p data_set is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pe_resource_t rsc, const pe_node_t node,
	pe_working_set_t *data_set)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (data_set == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, data_set);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pe_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pe_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)) {
	pcmk__config_warn("Resetting \"" XML_RSC_ATTR_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pe__set_resource_flags(rsc,
	pe_rsc_needs_fencing\|pe_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pe_native)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags, pe_flag_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pe_rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pe_resource_t *rsc)
	{
	const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pe_warn_once(pe_wo_upstart,
	"Support for Upstart resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pe_warn_once(pe_wo_nagios,
	"Support for Nagios resources (such as %s) is deprecated "
	"and will be removed in a future release of Pacemaker",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pe_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] data_set Cluster working set
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pe_resource_t *rsc,
	pe_resource_t parent, pe_working_set_t data_set)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = RSC_ROLE_UNKNOWN,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (data_set != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = data_set->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	if (id == NULL) {
	pe_err("Ignoring <%s> configuration without " XML_ATTR_ID,
	crm_element_name(xml_obj));
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, data_set) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pe_resource_t));
	if (*rsc == NULL) {
	crm_crit("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = data_set;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, data_set->input);

	(rsc)->variant = get_resource_type(crm_element_name((rsc)->xml));
	if ((*rsc)->variant == pe_unknown) {
	pe_err("Ignoring resource '%s' of unknown type '%s'",
	id, crm_element_name((*rsc)->xml));
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, data_set);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, data_set); // \deprecated

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pe_rsc_runnable\|pe_rsc_provisional);

	if (!pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
	pe__set_resource_flags(*rsc, pe_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = RSC_ROLE_STOPPED;
	(*rsc)->next_role = RSC_ROLE_UNKNOWN;

	- (*rsc)->recovery_type = recovery_stop_start;
	+ (*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_allow_migrate);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* allow-migrate=false.
	*/
	pe__set_resource_flags(*rsc, pe_rsc_allow_migrate);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	}
	if (pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
	pe__clear_resource_flags(*rsc, pe_rsc_managed);
	pe__set_resource_flags(*rsc, pe_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pe_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pe_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pe_rsc_unique);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pe_rsc_trace((*rsc), "%s dependency restart handling: restart",
	(*rsc)->id);
	pe_warn_once(pe_wo_restart_type,
	"Support for restart-type is deprecated and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pe_rsc_trace((*rsc), "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	- (*rsc)->recovery_type = recovery_stop_only;
	+ (*rsc)->recovery_type = pcmk_multiply_active_stop;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	- (*rsc)->recovery_type = recovery_block;
	+ (*rsc)->recovery_type = pcmk_multiply_active_block;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	- (*rsc)->recovery_type = recovery_stop_unexpected;
	+ (*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pe_warn("%s is not a valid value for " XML_RSC_ATTR_MULTIPLE
	", using default of \"stop_start\"", value);
	}
	- (*rsc)->recovery_type = recovery_stop_start;
	+ (*rsc)->recovery_type = pcmk_multiply_active_restart;
	pe_rsc_trace((*rsc), "%s multiple running resource recovery: "
	"stop/start", (*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pe_warn_once(pe_wo_neg_threshold,
	XML_RSC_ATTR_FAIL_STICKINESS
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pe__set_working_set_flags(data_set, pe_flag_have_stonith_resource);
	pe__set_resource_flags(*rsc, pe_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_TIMEOUT);
	if (value != NULL) {
	// Stored as seconds
	(*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, data_set);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* reconnect_interval.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	(*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
	/* we want to override any default failure_timeout in use when remote
	* reconnect_interval is in use. */
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pe_rsc_trace((rsc), "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != RSC_ROLE_UNKNOWN ? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, data_set) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", data_set);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default", data_set);
	}

	pe_rsc_trace((rsc), "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pe_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, data_set);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, data_set) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pe_resource_t child, pe_resource_t rsc)
	{
	pe_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pe_resource_t *
	uber_parent(pe_resource_t * rsc)
	{
	pe_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL && parent->parent->variant != pe_container) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pe_resource_t *
	pe__const_top_resource(const pe_resource_t *rsc, bool include_bundle)
	{
	const pe_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle && (parent->parent->variant == pe_container)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pe_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pe_rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pe_rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pe_resource_t rsc, pe_node_t node,
	pe_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of resource_object_functions_t:active_node()
	static pe_node_t *
	active_node(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pe_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pe_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pe_node_t *
	pe__find_active_requires(const pe_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pe_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pe_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)
	\|\| (rsc->role > RSC_ROLE_STOPPED)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pe_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pe_rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role), why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 04f05039ea..951b5b95e4 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1430 +1,1432 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pe_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pe_native) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pe_resource_t * rsc, pe_node_t * node, gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == RSC_ROLE_PROMOTED) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pe_rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pe__node_name(node), node->details->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pe_rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s) "
	"from guest node %s",
	pe__node_name(a_node), a_node->details->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == RSC_ROLE_PROMOTED)? " + 1" : "",
	pe__node_name(node));
	}
	}
	}

	void
	native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pe_rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
	pcmk_is_set(rsc->flags, pe_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pe_native) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if (rsc->variant == pe_native && node->details->maintenance) {
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	pe__set_resource_flags(rsc, pe_rsc_maintenance);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_resource_t *p = rsc->parent;

	pe_rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default", data_set);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	- case recovery_stop_only:
	+ case pcmk_multiply_active_stop:
	{
	GHashTableIter gIter;
	pe_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(data_set->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	- case recovery_block:
	+ case pcmk_multiply_active_block:
	pe__clear_resource_flags(rsc, pe_rsc_managed);
	pe__set_resource_flags(rsc, pe_rsc_block);

	/* If the resource belongs to a group or bundle configured with
	* multiple-active=block, block the entire entity.
	*/
	if (rsc->parent
	&& (rsc->parent->variant == pe_group \|\| rsc->parent->variant == pe_container)
	- && rsc->parent->recovery_type == recovery_block) {
	+ && (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	pe__clear_resource_flags(child, pe_rsc_managed);
	pe__set_resource_flags(child, pe_rsc_block);
	}
	}
	break;
	- default: // recovery_stop_start, recovery_stop_unexpected
	+
	+ // pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
	+ default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pe__node_name(node),
	recovery2text(rsc->recovery_type));

	} else {
	pe_rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pe__node_name(node));
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, data_set, FALSE);
	}
	}

	static void
	recursive_clear_unique(pe_resource_t *rsc, gpointer user_data)
	{
	pe__clear_resource_flags(rsc, pe_rsc_unique);
	add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	pe_resource_t *parent = uber_parent(rsc);
	const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pe_rsc_unique) && pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, data_set);

	/* Clear globally-unique on the parent and all its descendants unpacked
	* so far (clearing the parent should make any future children unpacking
	* correct). We have to clear this resource explicitly because it isn't
	* hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pe_rsc_promotable)) {

	pe_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pe_resource_t rsc, const pe_node_t node, int flags)
	{
	pe_rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pe__node_name(node));

	if (pcmk_is_set(flags, pe_find_current) && rsc->running_on) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pe_node_t loc = (pe_node_t ) iter->data;

	if (loc->details == node->details) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive)
	&& (rsc->running_on == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pe_find_current) && rsc->allocated_to
	&& (rsc->allocated_to->details == node->details)) {
	return true;
	}
	return false;
	}

	pe_resource_t *
	native_find_rsc(pe_resource_t * rsc, const char id, const pe_node_t on_node,
	int flags)
	{
	bool match = false;
	pe_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (flags & pe_find_clone) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pe_find_renamed)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = true;

	} else if (pcmk_is_set(flags, pe_find_any)
	\|\| (pcmk_is_set(flags, pe_find_anon)
	&& !pcmk_is_set(rsc->flags, pe_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, data_set);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	pcmk__str_update(&value_copy, value);
	return value_copy;
	}

	gboolean
	native_active(pe_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pe_node_t a_node = (pe_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pe_rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	} else if (a_node->details->online == FALSE && pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pe__node_name(a_node));
	} else {
	pe_rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pe_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_STOP, pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_MIGRATED, pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_PROMOTE, pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_DEMOTE, pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(const pe_resource_t *rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, CRMD_ACTION_STATUS, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(const pe_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == RSC_ROLE_STARTED)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pe_rsc_promotable)) {

	role = RSC_ROLE_UNPROMOTED;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pe_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	native_print_xml(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print(XML_ATTR_ID "=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_orphan));
	status_print("blocked=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_block));
	status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pe_rsc_failure_ignored));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" " XML_ATTR_ID "=\"%s\" "
	"cached=\"%s\"/>\n",
	pre_text, pcmk__s(node->details->uname, ""),
	node->details->id, pcmk__btoa(!node->details->online));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pe_resource_t rsc, const char name,
	const pe_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pe_native) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > RSC_ROLE_UNPROMOTED) {
	pcmk__add_word(&outstr, 0, role2text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pe__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == RSC_ROLE_STOPPED) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent the
	* resource from becoming promoted).
	*/
	if (target_role_e == RSC_ROLE_STOPPED) {
	have_flags = add_output_flag(outstr, "disabled", have_flags);

	} else if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pe_rsc_promotable)
	&& target_role_e == RSC_ROLE_UNPROMOTED) {
	have_flags = add_output_flag(outstr, "target-role:", have_flags);
	g_string_append(outstr, target_role);
	}
	}

	// Blocked or maintenance implies unmanaged
	if (pcmk_any_flags_set(rsc->flags, pe_rsc_block\|pe_rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t n = (pe_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pe_resource_t rsc,
	const char name, const pe_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pe_native);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	cl = "rsc-failed";

	} else if (rsc->variant == pe_native && (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pe_resource_t rsc,
	const char name, const pe_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	common_print(pe_resource_t rsc, const char pre_text, const char *name,
	const pe_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	XML_RSC_ATTR_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t n = (pe_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", pe__node_name(n));

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", pe__node_name(n));

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, pe__node_name(n));

	} else {
	status_print("%s", pe__node_name(n));
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	native_print(pe_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const pe_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pe_native);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	const char *desc = NULL;
	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	const char *lock_node_name = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, XML_ATTR_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));

	if (rsc->lock_node != NULL) {
	lock_node_name = rsc->lock_node->details->uname;
	}

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
	"id", rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pe_rsc_orphan),
	"blocked", pe__rsc_bool_str(rsc, pe_rsc_block),
	"maintenance", pe__rsc_bool_str(rsc, pe_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pe_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pe_rsc_failure_ignored),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL),
	"locked_to", lock_node_name,
	"description", desc);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	"name", node->details->uname,
	"id", node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pe_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pe_native);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pe_resource_t ", "GList ", "GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pe_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pe_native);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pe_resource_t * rsc)
	{
	pe_rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pe_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pe_node_t *
	native_location(const pe_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pe__rsc_node argument instead of int current
	pe_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pe_native) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_node_t node = (pe_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pe_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	const pe_resource_t *up = pe__const_top_resource(rsc, true);

	return up->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pe_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pe_native));
	return 1U;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jul 20, 8:37 PM (15 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2081426
Default Alt Text: (159 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions