No OneTemporary
Actions

Size

361 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h
	index 43a7ae8542..c70e95a9de 100644
	--- a/include/crm/common/xml_internal.h
	+++ b/include/crm/common/xml_internal.h
	@@ -1,561 +1,575 @@
	/*
	* Copyright 2017-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__XML_INTERNAL__H
	# define PCMK__XML_INTERNAL__H

	/*
	* Internal-only wrappers for and extensions to libxml2 (libxslt)
	*/

	# include <stdlib.h>
	# include <stdio.h>
	# include <string.h>

	# include <crm/crm.h> /* transitively imports qblog.h */
	# include <crm/common/output_internal.h>
	# include <crm/common/xml_io_internal.h>
	+# include <crm/common/xml_names_internal.h> // PCMK__XE_PROMOTABLE_LEGACY

	# include <libxml/relaxng.h>

	/*!
	* \brief Base for directing lib{xml2,xslt} log into standard libqb backend
	*
	* This macro implements the core of what can be needed for directing
	* libxml2 or libxslt error messaging into standard, preconfigured
	* libqb-backed log stream.
	*
	* It's a bit unfortunate that libxml2 (and more sparsely, also libxslt)
	* emits a single message by chunks (location is emitted separatedly from
	* the message itself), so we have to take the effort to combine these
	* chunks back to single message. Whether to do this or not is driven
	* with \p dechunk toggle.
	*
	* The form of a macro was chosen for implicit deriving of __FILE__, etc.
	* and also because static dechunking buffer should be differentiated per
	* library (here we assume different functions referring to this macro
	* will not ever be using both at once), preferably also per-library
	* context of use to avoid clashes altogether.
	*
	* Note that we cannot use qb_logt, because callsite data have to be known
	* at the moment of compilation, which it is not always the case -- xml_log
	* (and unfortunately there's no clear explanation of the fail to compile).
	*
	* Also note that there's no explicit guard against said libraries producing
	* never-newline-terminated chunks (which would just keep consuming memory),
	* as it's quite improbable. Termination of the program in between the
	* same-message chunks will raise a flag with valgrind and the likes, though.
	*
	* And lastly, regarding how dechunking combines with other non-message
	* parameters -- for \p priority, most important running specification
	* wins (possibly elevated to LOG_ERR in case of nonconformance with the
	* newline-termination "protocol"), \p dechunk is expected to always be
	* on once it was at the start, and the rest (\p postemit and \p prefix)
	* are picked directly from the last chunk entry finalizing the message
	* (also reasonable to always have it the same with all related entries).
	*
	* \param[in] priority Syslog priority for the message to be logged
	* \param[in] dechunk Whether to dechunk new-line terminated message
	* \param[in] postemit Code to be executed once message is sent out
	* \param[in] prefix How to prefix the message or NULL for raw passing
	* \param[in] fmt Format string as with printf-like functions
	* \param[in] ap Variable argument list to supplement \p fmt format string
	*/
	#define PCMK__XML_LOG_BASE(priority, dechunk, postemit, prefix, fmt, ap) \
	do { \
	if (!(dechunk) && (prefix) == NULL) { /* quick pass */ \
	qb_log_from_external_source_va(__func__, __FILE__, (fmt), \
	(priority), __LINE__, 0, (ap)); \
	(void) (postemit); \
	} else { \
	int CXLB_len = 0; \
	char *CXLB_buf = NULL; \
	static int CXLB_buffer_len = 0; \
	static char *CXLB_buffer = NULL; \
	static uint8_t CXLB_priority = 0; \
	\
	CXLB_len = vasprintf(&CXLB_buf, (fmt), (ap)); \
	\
	if (CXLB_len <= 0 \|\| CXLB_buf[CXLB_len - 1] == '\n' \|\| !(dechunk)) { \
	if (CXLB_len < 0) { \
	CXLB_buf = (char ) "LOG CORRUPTION HAZARD"; /we don't modify*/\
	CXLB_priority = QB_MIN(CXLB_priority, LOG_ERR); \
	} else if (CXLB_len > 0 /* && (dechunk) */ \
	&& CXLB_buf[CXLB_len - 1] == '\n') { \
	CXLB_buf[CXLB_len - 1] = '\0'; \
	} \
	if (CXLB_buffer) { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s%s", \
	CXLB_priority, __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buffer, CXLB_buf); \
	free(CXLB_buffer); \
	} else { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s", \
	(priority), __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buf); \
	} \
	if (CXLB_len < 0) { \
	CXLB_buf = NULL; /* restore temporary override */ \
	} \
	CXLB_buffer = NULL; \
	CXLB_buffer_len = 0; \
	(void) (postemit); \
	\
	} else if (CXLB_buffer == NULL) { \
	CXLB_buffer_len = CXLB_len; \
	CXLB_buffer = CXLB_buf; \
	CXLB_buf = NULL; \
	CXLB_priority = (priority); /* remember as a running severest */ \
	\
	} else { \
	CXLB_buffer = realloc(CXLB_buffer, 1 + CXLB_buffer_len + CXLB_len); \
	memcpy(CXLB_buffer + CXLB_buffer_len, CXLB_buf, CXLB_len); \
	CXLB_buffer_len += CXLB_len; \
	CXLB_buffer[CXLB_buffer_len] = '\0'; \
	CXLB_priority = QB_MIN(CXLB_priority, (priority)); /* severest? */ \
	} \
	free(CXLB_buf); \
	} \
	} while (0)

	/*
	* \enum pcmk__xml_fmt_options
	* \brief Bit flags to control format in XML logs and dumps
	*/
	enum pcmk__xml_fmt_options {
	//! Exclude certain XML attributes (for calculating digests)
	pcmk__xml_fmt_filtered = (1 << 0),

	//! Include indentation and newlines
	pcmk__xml_fmt_pretty = (1 << 1),

	//! Include the opening tag of an XML element, and include XML comments
	pcmk__xml_fmt_open = (1 << 3),

	//! Include the children of an XML element
	pcmk__xml_fmt_children = (1 << 4),

	//! Include the closing tag of an XML element
	pcmk__xml_fmt_close = (1 << 5),

	// @COMPAT Can we start including text nodes unconditionally?
	//! Include XML text nodes
	pcmk__xml_fmt_text = (1 << 6),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a created XML subtree
	pcmk__xml_fmt_diff_plus = (1 << 7),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a removed XML subtree
	pcmk__xml_fmt_diff_minus = (1 << 8),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a minimal version of an XML diff (only showing the changes)
	pcmk__xml_fmt_diff_short = (1 << 9),
	};

	int pcmk__xml_show(pcmk__output_t out, const char prefix, const xmlNode *data,
	int depth, uint32_t options);
	int pcmk__xml_show_changes(pcmk__output_t out, const xmlNode xml);

	/* XML search strings for guest, remote and pacemaker_remote nodes */

	/* search string to find CIB resources entries for cluster nodes */
	#define PCMK__XP_MEMBER_NODE_CONFIG \
	"//" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" PCMK_XE_NODE "[not(@type) or @type='member']"

	/* search string to find CIB resources entries for guest nodes */
	#define PCMK__XP_GUEST_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"//" PCMK_XE_META_ATTRIBUTES "//" PCMK_XE_NVPAIR \
	"[@name='" PCMK_META_REMOTE_NODE "']"

	/* search string to find CIB resources entries for remote nodes */
	#define PCMK__XP_REMOTE_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"[@type='remote'][@provider='pacemaker']"

	/* search string to find CIB node status entries for pacemaker_remote nodes */
	#define PCMK__XP_REMOTE_NODE_STATUS \
	"//" PCMK_XE_CIB "//" PCMK_XE_STATUS "//" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_REMOTE_NODE "='true']"
	/*!
	* \internal
	* \brief Serialize XML (using libxml) into provided descriptor
	*
	* \param[in] fd File descriptor to (piece-wise) write to
	* \param[in] cur XML subtree to proceed
	*
	* \return a standard Pacemaker return code
	*/
	int pcmk__xml2fd(int fd, xmlNode *cur);

	enum pcmk__xml_artefact_ns {
	pcmk__xml_artefact_ns_legacy_rng = 1,
	pcmk__xml_artefact_ns_legacy_xslt,
	pcmk__xml_artefact_ns_base_rng,
	pcmk__xml_artefact_ns_base_xslt,
	};

	void pcmk__strip_xml_text(xmlNode *xml);
	const char pcmk__xe_add_last_written(xmlNode xe);

	xmlNode pcmk__xe_first_child(const xmlNode parent, const char *node_name,
	const char attr_n, const char attr_v);


	void pcmk__xe_remove_attr(xmlNode element, const char name);
	void pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data);
	int pcmk__xe_delete_match(xmlNode xml, xmlNode search);
	int pcmk__xe_replace_match(xmlNode xml, xmlNode replace);
	int pcmk__xe_update_match(xmlNode xml, xmlNode update);

	GString pcmk__element_xpath(const xmlNode xml);

	/*!
	* \internal
	* \enum pcmk__xml_escape_type
	* \brief Indicators of which XML characters to escape
	*
	* XML allows the escaping of special characters by replacing them with entity
	* references (for example, <tt>"""</tt>) or character references (for
	* example, <tt>" "</tt>).
	*
	* The special characters <tt>'&'</tt> (except as the beginning of an entity
	* reference) and <tt>'<'</tt> are not allowed in their literal forms in XML
	* character data. Character data is non-markup text (for example, the content
	* of a text node). <tt>'>'</tt> is allowed under most circumstances; we escape
	* it for safety and symmetry.
	*
	* For more details, see the "Character Data and Markup" section of the XML
	* spec, currently section 2.4:
	* https://www.w3.org/TR/xml/#dt-markup
	*
	* Attribute values are handled specially.
	* * If an attribute value is delimited by single quotes, then single quotes
	* must be escaped within the value.
	* * Similarly, if an attribute value is delimited by double quotes, then double
	* quotes must be escaped within the value.
	* * A conformant XML processor replaces a literal whitespace character (tab,
	* newline, carriage return, space) in an attribute value with a space
	* (\c '#x20') character. However, a reference to a whitespace character (for
	* example, \c " " for \c '\n') does not get replaced.
	* * For more details, see the "Attribute-Value Normalization" section of the
	* XML spec, currently section 3.3.3. Note that the default attribute type
	* is CDATA; we don't deal with NMTOKENS, etc.:
	* https://www.w3.org/TR/xml/#AVNormalize
	*
	* Pacemaker always delimits attribute values with double quotes, so there's no
	* need to escape single quotes.
	*
	* Newlines and tabs should be escaped in attribute values when XML is
	* serialized to text, so that future parsing preserves them rather than
	* normalizing them to spaces.
	*
	* We always escape carriage returns, so that they're not converted to spaces
	* during attribute-value normalization and because displaying them as literals
	* is messy.
	*/
	enum pcmk__xml_escape_type {
	/*!
	* For text nodes.
	* * Escape \c '<', \c '>', and \c '&' using entity references.
	* * Do not escape \c '\n' and \c '\t'.
	* * Escape other non-printing characters using character references.
	*/
	pcmk__xml_escape_text,

	/*!
	* For attribute values.
	* * Escape \c '<', \c '>', \c '&', and \c '"' using entity references.
	* * Escape \c '\n', \c '\t', and other non-printing characters using
	* character references.
	*/
	pcmk__xml_escape_attr,

	/* @COMPAT Drop escaping of at least '\n' and '\t' for
	* pcmk__xml_escape_attr_pretty when openstack-info, openstack-floating-ip,
	* and openstack-virtual-ip resource agents no longer depend on it.
	*
	* At time of writing, openstack-info may set a multiline value for the
	* openstack_ports node attribute. The other two agents query the value and
	* require it to be on one line with no spaces.
	*/
	/*!
	* For attribute values displayed in text output delimited by double quotes.
	* * Escape \c '\n' as \c "\\n"
	* * Escape \c '\r' as \c "\\r"
	* * Escape \c '\t' as \c "\\t"
	* * Escape \c '"' as \c "\\""
	*/
	pcmk__xml_escape_attr_pretty,
	};

	bool pcmk__xml_needs_escape(const char *text, enum pcmk__xml_escape_type type);
	char pcmk__xml_escape(const char text, enum pcmk__xml_escape_type type);

	/*!
	* \internal
	* \brief Get the root directory to scan XML artefacts of given kind for
	*
	* \param[in] ns governs the hierarchy nesting against the inherent root dir
	*
	* \return root directory to scan XML artefacts of given kind for
	*/
	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns);

	/*!
	* \internal
	* \brief Get the fully unwrapped path to particular XML artifact (RNG/XSLT)
	*
	* \param[in] ns denotes path forming details (parent dir, suffix)
	* \param[in] filespec symbolic file specification to be combined with
	* #artefact_ns to form the final path
	* \return unwrapped path to particular XML artifact (RNG/XSLT)
	*/
	char *pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns,
	const char *filespec);

	/*!
	* \internal
	* \brief Retrieve the value of the \c PCMK_XA_ID XML attribute
	*
	* \param[in] xml XML element to check
	*
	* \return Value of the \c PCMK_XA_ID attribute (may be \c NULL)
	*/
	static inline const char *
	pcmk__xe_id(const xmlNode *xml)
	{
	return crm_element_value(xml, PCMK_XA_ID);
	}

	/*!
	* \internal
	* \brief Check whether an XML element is of a particular type
	*
	* \param[in] xml XML element to compare
	* \param[in] name XML element name to compare
	*
	* \return \c true if \p xml is of type \p name, otherwise \c false
	*/
	static inline bool
	pcmk__xe_is(const xmlNode xml, const char name)
	{
	return (xml != NULL) && (xml->name != NULL) && (name != NULL)
	&& (strcmp((const char *) xml->name, name) == 0);
	}

	/*!
	* \internal
	* \brief Return first non-text child node of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First non-text child node of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type == XML_TEXT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling node of an XML node
	*
	* \param[in] child XML node to check
	*
	* \return Next non-text sibling of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_next(const xmlNode *child)
	{
	xmlNode *next = (child? child->next : NULL);

	while (next && (next->type == XML_TEXT_NODE)) {
	next = next->next;
	}
	return next;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling element of an XML element
	*
	* \param[in] child XML element to check
	*
	* \return Next sibling element of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_next(const xmlNode *child)
	{
	xmlNode *next = child? child->next : NULL;

	while (next && (next->type != XML_ELEMENT_NODE)) {
	next = next->next;
	}
	return next;
	}

	xmlNode pcmk__xe_create(xmlNode parent, const char *name);
	xmlNode pcmk__xml_copy(xmlNode parent, xmlNode *src);
	xmlNode pcmk__xe_next_same(const xmlNode node);

	void pcmk__xe_set_content(xmlNode node, const char format, ...)
	G_GNUC_PRINTF(2, 3);

	/*!
	* \internal
	* \brief Like pcmk__xe_set_props, but takes a va_list instead of
	* arguments directly.
	*
	* \param[in,out] node XML to add attributes to
	* \param[in] pairs NULL-terminated list of name/value pairs to add
	*/
	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs);

	/*!
	* \internal
	* \brief Add a NULL-terminated list of name/value pairs to the given
	* XML node as properties.
	*
	* \param[in,out] node XML node to add properties to
	* \param[in] ... NULL-terminated list of name/value pairs
	*
	* \note A NULL name terminates the arguments; a NULL value will be skipped.
	*/
	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	G_GNUC_NULL_TERMINATED;

	/*!
	* \internal
	* \brief Get first attribute of an XML element
	*
	* \param[in] xe XML element to check
	*
	* \return First attribute of \p xe (or NULL if \p xe is NULL or has none)
	*/
	static inline xmlAttr *
	pcmk__xe_first_attr(const xmlNode *xe)
	{
	return (xe == NULL)? NULL : xe->properties;
	}

	/*!
	* \internal
	* \brief Extract the ID attribute from an XML element
	*
	* \param[in] xpath String to search
	* \param[in] node Node to get the ID for
	*
	* \return ID attribute of \p node in xpath string \p xpath
	*/
	char *
	pcmk__xpath_node_id(const char xpath, const char node);

	/*!
	* \internal
	* \brief Print an informational message if an xpath query returned multiple
	* items with the same ID.
	*
	* \param[in,out] out The output object
	* \param[in] search The xpath search result, most typically the result of
	* calling cib->cmds->query().
	* \param[in] name The name searched for
	*/
	void
	pcmk__warn_multiple_name_matches(pcmk__output_t out, xmlNode search,
	const char *name);

	/* internal XML-related utilities */

	enum xml_private_flags {
	pcmk__xf_none = 0x0000,
	pcmk__xf_dirty = 0x0001,
	pcmk__xf_deleted = 0x0002,
	pcmk__xf_created = 0x0004,
	pcmk__xf_modified = 0x0008,

	pcmk__xf_tracking = 0x0010,
	pcmk__xf_processed = 0x0020,
	pcmk__xf_skip = 0x0040,
	pcmk__xf_moved = 0x0080,

	pcmk__xf_acl_enabled = 0x0100,
	pcmk__xf_acl_read = 0x0200,
	pcmk__xf_acl_write = 0x0400,
	pcmk__xf_acl_deny = 0x0800,

	pcmk__xf_acl_create = 0x1000,
	pcmk__xf_acl_denied = 0x2000,
	pcmk__xf_lazy = 0x4000,
	};

	void pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag);

	/*!
	* \internal
	* \brief Iterate over child elements of \p xml
	*
	* This function iterates over the children of \p xml, performing the
	* callback function \p handler on each node. If the callback returns
	* a value other than pcmk_rc_ok, the iteration stops and the value is
	* returned. It is therefore possible that not all children will be
	* visited.
	*
	* \param[in,out] xml The starting XML node. Can be NULL.
	* \param[in] child_element_name The name that the node must match in order
	* for \p handler to be run. If NULL, all
	* child elements will match.
	* \param[in] handler The callback function.
	* \param[in,out] userdata User data to pass to the callback function.
	* Can be NULL.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata);

	static inline const char *
	pcmk__xml_attr_value(const xmlAttr *attr)
	{
	return ((attr == NULL) \|\| (attr->children == NULL))? NULL
	: (const char *) attr->children->content;
	}


	gboolean pcmk__validate_xml(xmlNode xml_blob, const char validation,
	xmlRelaxNGValidityErrorFunc error_handler,
	void *error_handler_context);

	void pcmk__log_known_schemas(void);
	const char *pcmk__remote_schema_dir(void);
	void pcmk__sort_schemas(void);

	// @COMPAT Remove when v1 patchsets are removed
	xmlNode pcmk__diff_v1_xml_object(xmlNode left, xmlNode *right, bool suppress);

	+// @COMPAT Drop when PCMK__XE_PROMOTABLE_LEGACY is removed
	+static inline const char *
	+pcmk__map_element_name(const xmlNode *xml)
	+{
	+ if (xml == NULL) {
	+ return NULL;
	+ } else if (pcmk__xe_is(xml, PCMK__XE_PROMOTABLE_LEGACY)) {
	+ return PCMK_XE_CLONE;
	+ } else {
	+ return (const char *) xml->name;
	+ }
	+}
	+
	#endif // PCMK__XML_INTERNAL__H
	diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
	index caa95616e8..35bfc9d29d 100644
	--- a/lib/pacemaker/pcmk_scheduler.c
	+++ b/lib/pacemaker/pcmk_scheduler.c
	@@ -1,892 +1,893 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	/*!
	* \internal
	* \brief Do deferred action checks after assignment
	*
	* When unpacking the resource history, the scheduler checks for resource
	* configurations that have changed since an action was run. However, at that
	* time, bundles using the REMOTE_CONTAINER_HACK don't have their final
	* parameter information, so instead they add a deferred check to a list. This
	* function processes one entry in that list.
	*
	* \param[in,out] rsc Resource that action history is for
	* \param[in,out] node Node that action history is for
	* \param[in] rsc_op Action history entry
	* \param[in] check Type of deferred check to do
	*/
	static void
	check_params(pcmk_resource_t rsc, pcmk_node_t node, const xmlNode *rsc_op,
	enum pcmk__check_parameters check)
	{
	const char *reason = NULL;
	pcmk__op_digest_t *digest_data = NULL;

	switch (check) {
	case pcmk__check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL)) {
	reason = "action definition changed";
	}
	break;

	case pcmk__check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s has "
	"no digest to compare",
	rsc->id, pcmk__xe_id(rsc_op), node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}
	if (reason != NULL) {
	pe__clear_failcount(rsc, node, reason, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has failcount clearing scheduled on a node
	*
	* \param[in] node Node to check
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc has failcount clearing scheduled on \p node,
	* otherwise false
	*/
	static bool
	failcount_clear_action_exists(const pcmk_node_t *node,
	const pcmk_resource_t *rsc)
	{
	GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
	TRUE);

	if (list != NULL) {
	g_list_free(list);
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it reached its failure threshold there
	*
	* \param[in,out] data Resource to check failure threshold for
	* \param[in] user_data Node to check resource on
	*/
	static void
	check_failure_threshold(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, check_failure_threshold, user_data);
	return;
	}

	if (!failcount_clear_action_exists(node, rsc)) {
	/* Don't force the resource away from this node due to a failcount
	* that's going to be cleared.
	*
	* @TODO Failcount clearing can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* schedule_resource_actions() via check_params(). This runs well before
	* then, so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't. Worst case, we stop or move the
	* resource, then move it back in the next transition.
	*/
	pcmk_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -PCMK_SCORE_INFINITY,
	"__fail_limit__", rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief If resource has exclusive discovery, ban node if not allowed
	*
	* Location constraints have a PCMK_XA_RESOURCE_DISCOVERY option that allows
	* users to specify where probes are done for the affected resource. If this is
	* set to \c exclusive, probes will only be done on nodes listed in exclusive
	* constraints. This function bans the resource from the node if the node is not
	* listed.
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Node to check resource on
	*/
	static void
	apply_exclusive_discovery(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	if (rsc->exclusive_discover
	\|\| pe__const_top_resource(rsc, false)->exclusive_discover) {
	pcmk_node_t *match = NULL;

	// If this is a collective resource, apply recursively to children
	g_list_foreach(rsc->children, apply_exclusive_discovery, user_data);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if ((match != NULL)
	&& (match->rsc_discover_mode != pcmk_probe_exclusive)) {
	match->weight = -PCMK_SCORE_INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Apply stickiness to a resource if appropriate
	*
	* \param[in,out] data Resource to check for stickiness
	* \param[in] user_data Ignored
	*/
	static void
	apply_stickiness(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	pcmk_node_t *node = NULL;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, apply_stickiness, NULL);
	return;
	}

	/* A resource is sticky if it is managed, has stickiness configured, and is
	* active on a single node.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| (rsc->stickiness < 1) \|\| !pcmk__list_of_1(rsc->running_on)) {
	return;
	}

	node = rsc->running_on->data;

	/* In a symmetric cluster, stickiness can always be used. In an
	* asymmetric cluster, we have to check whether the resource is still
	* allowed on the node, so we don't keep the resource somewhere it is no
	* longer explicitly enabled.
	*/
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id) == NULL)) {
	pcmk__rsc_debug(rsc,
	"Ignoring %s stickiness because the cluster is "
	"asymmetric and %s is not explicitly allowed",
	rsc->id, pcmk__node_name(node));
	return;
	}

	pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
	rsc->id, rsc->stickiness, pcmk__node_name(node));
	resource_location(rsc, node, rsc->stickiness, "stickiness", rsc->cluster);
	}

	/*!
	* \internal
	* \brief Apply shutdown locks for all resources as appropriate
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	apply_shutdown_locks(pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	return;
	}
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->shutdown_lock(rsc);
	}
	}

	/*!
	* \internal
	* \brief Calculate the number of available nodes in the cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	count_available_nodes(pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(scheduler->flags, pcmk_sched_no_compat)) {
	return;
	}

	// @COMPAT for API backward compatibility only (cluster does not use value)
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if ((node != NULL) && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	scheduler->max_valid_nodes++;
	}
	}
	crm_trace("Online node count: %d", scheduler->max_valid_nodes);
	}

	/*
	* \internal
	* \brief Apply node-specific scheduling criteria
	*
	* After the CIB has been unpacked, process node-specific scheduling criteria
	* including shutdown locks, location constraints, resource stickiness,
	* migration thresholds, and exclusive resource discovery.
	*/
	static void
	apply_node_criteria(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Applying node-specific scheduling criteria");
	apply_shutdown_locks(scheduler);
	count_available_nodes(scheduler);
	pcmk__apply_locations(scheduler);
	g_list_foreach(scheduler->resources, apply_stickiness, NULL);

	for (GList *node_iter = scheduler->nodes; node_iter != NULL;
	node_iter = node_iter->next) {
	for (GList *rsc_iter = scheduler->resources; rsc_iter != NULL;
	rsc_iter = rsc_iter->next) {
	check_failure_threshold(rsc_iter->data, node_iter->data);
	apply_exclusive_discovery(rsc_iter->data, node_iter->data);
	}
	}
	}

	/*!
	* \internal
	* \brief Assign resources to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	assign_resources(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;

	crm_trace("Assigning resources to nodes");

	if (!pcmk__str_eq(scheduler->placement_strategy, PCMK_VALUE_DEFAULT,
	pcmk__str_casei)) {
	pcmk__sort_resources(scheduler);
	}
	pcmk__show_node_capacities("Original", scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	/* Assign remote connection resources first (which will also assign any
	* colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
	rsc->id);
	rsc->cmds->assign(rsc, rsc->partial_migration_target, true);
	}
	}
	}

	/* now do the rest of the resources */
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (!rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
	rsc->xml->name, rsc->id);
	rsc->cmds->assign(rsc, NULL, true);
	}
	}

	pcmk__show_node_capacities("Remaining", scheduler);
	}

	/*!
	* \internal
	* \brief Schedule fail count clearing on online nodes if resource is orphaned
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Ignored
	*/
	static void
	clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	return;
	}
	crm_trace("Clear fail counts for orphaned resource %s", rsc->id);

	/* There's no need to recurse into rsc->children because those
	* should just be unassigned clone instances.
	*/

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *clear_op = NULL;

	if (!node->details->online) {
	continue;
	}
	if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
	continue;
	}

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	rsc->cluster);

	/* We can't use order_action_then_stop() here because its
	* pcmk__ar_guest_allowed breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
	NULL, pcmk__ar_ordered, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Schedule any resource actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_resource_actions(pcmk_scheduler_t *scheduler)
	{
	// Process deferred action checks
	pe__foreach_param_check(scheduler, check_params);
	pe__free_param_checks(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_trace("Scheduling probes");
	pcmk__schedule_probes(scheduler);
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	g_list_foreach(scheduler->resources, clear_failcounts_if_orphaned,
	NULL);
	}

	crm_trace("Scheduling resource actions");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->create_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its descendants are managed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if resource or any descendant is managed, otherwise false
	*/
	static bool
	is_managed(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (is_managed((pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether any resources in the cluster are managed
	*
	* \param[in] scheduler Scheduler data
	*
	* \return true if any resource is managed, otherwise false
	*/
	static bool
	any_managed_resources(const pcmk_scheduler_t *scheduler)
	{
	for (const GList *iter = scheduler->resources;
	iter != NULL; iter = iter->next) {
	if (is_managed((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node requires fencing
	*
	* \param[in] node Node to check
	* \param[in] have_managed Whether any resource in cluster is managed
	*
	* \return true if \p node should be fenced, otherwise false
	*/
	static bool
	needs_fencing(const pcmk_node_t *node, bool have_managed)
	{
	return have_managed && node->details->unclean
	&& pe_can_fence(node->details->data_set, node);
	}

	/*!
	* \internal
	* \brief Check whether a node requires shutdown
	*
	* \param[in] node Node to check
	*
	* \return true if \p node should be shut down, otherwise false
	*/
	static bool
	needs_shutdown(const pcmk_node_t *node)
	{
	if (pcmk__is_pacemaker_remote_node(node)) {
	/* Do not send shutdown actions for Pacemaker Remote nodes.
	* @TODO We might come up with a good use for this in the future.
	*/
	return false;
	}
	return node->details->online && node->details->shutdown;
	}

	/*!
	* \internal
	* \brief Track and order non-DC fencing
	*
	* \param[in,out] list List of existing non-DC fencing actions
	* \param[in,out] action Fencing action to prepend to \p list
	* \param[in] scheduler Scheduler data
	*
	* \return (Possibly new) head of \p list
	*/
	static GList *
	add_nondc_fencing(GList list, pcmk_action_t action,
	const pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)
	&& (list != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
	}
	return g_list_prepend(list, action);
	}

	/*!
	* \internal
	* \brief Schedule a node for fencing
	*
	* \param[in,out] node Node that requires fencing
	*/
	static pcmk_action_t *
	schedule_fencing(pcmk_node_t *node)
	{
	pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
	FALSE, node->details->data_set);

	pcmk__sched_warn("Scheduling node %s for fencing", pcmk__node_name(node));
	pcmk__order_vs_fence(fencing, node->details->data_set);
	return fencing;
	}

	/*!
	* \internal
	* \brief Create and order node fencing and shutdown actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *dc_down = NULL;
	bool integrity_lost = false;
	bool have_managed = any_managed_resources(scheduler);
	GList *fencing_ops = NULL;
	GList *shutdown_ops = NULL;

	crm_trace("Scheduling fencing and shutdowns as needed");
	if (!have_managed) {
	crm_notice("No fencing will be done until there are resources "
	"to manage");
	}

	// Check each node for whether it needs fencing or shutdown
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *fencing = NULL;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pcmk__is_guest_or_bundle_node(node)) {
	if (node->details->remote_requires_reset && have_managed
	&& pe_can_fence(scheduler, node)) {
	pcmk__fence_guest(node);
	}
	continue;
	}

	if (needs_fencing(node, have_managed)) {
	fencing = schedule_fencing(node);

	// Track DC and non-DC fence actions separately
	if (node->details->is_dc) {
	dc_down = fencing;
	} else {
	fencing_ops = add_nondc_fencing(fencing_ops, fencing,
	scheduler);
	}

	} else if (needs_shutdown(node)) {
	pcmk_action_t *down_op = pcmk__new_shutdown_action(node);

	// Track DC and non-DC shutdown actions separately
	if (node->details->is_dc) {
	dc_down = down_op;
	} else {
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if ((fencing == NULL) && node->details->unclean) {
	integrity_lost = true;
	pcmk__config_warn("Node %s is unclean but cannot be fenced",
	pcmk__node_name(node));
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resource functionality and data integrity "
	"cannot be guaranteed (configure, enable, "
	"and test fencing to correct this)");

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	crm_notice("Unclean nodes will not be fenced until quorum is "
	"attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
	PCMK_VALUE_IGNORE);
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	pcmk__order_after_each(dc_down, shutdown_ops);
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	pcmk__order_after_each(dc_down, fencing_ops);
	} else if (fencing_ops != NULL) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
	pcmk__ar_ordered);
	}
	}
	g_list_free(fencing_ops);
	g_list_free(shutdown_ops);
	}

	static void
	log_resource_details(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;

	/* Due to the `crm_mon --node=` feature, out->message() for all the
	* resource-related messages expects a list of nodes that we are allowed to
	* output information for. Here, we create a wildcard to match all nodes.
	*/
	all = g_list_prepend(all, (gpointer) "*");

	for (GList *item = scheduler->resources; item != NULL; item = item->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) item->data;

	// Log all resources except inactive orphans
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role != pcmk_role_stopped)) {
	- out->message(out, crm_map_element_name(rsc->xml), 0UL, rsc, all, all);
	+ out->message(out, pcmk__map_element_name(rsc->xml), 0UL, rsc, all,
	+ all);
	}
	}

	g_list_free(all);
	}

	static void
	log_all_actions(pcmk_scheduler_t *scheduler)
	{
	/* This only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	pcmk__output_t *prev_out = scheduler->priv;
	pcmk__output_t *out = NULL;

	if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
	return;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	pcmk__output_set_log_level(out, LOG_NOTICE);
	scheduler->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	scheduler->priv = prev_out;
	}

	/*!
	* \internal
	* \brief Log all required but unrunnable actions at trace level
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
	{
	const uint64_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	crm_trace("Required but unrunnable actions:");
	for (const GList *iter = scheduler->actions;
	iter != NULL; iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (!pcmk_any_flags_set(action->flags, flags)) {
	pcmk__log_action("\t", action, true);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack the CIB for scheduling
	*
	* \param[in,out] cib CIB XML to unpack (may be NULL if already unpacked)
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_cib(xmlNode cib, unsigned long long flags, pcmk_scheduler_t scheduler)
	{
	const char* localhost_save = NULL;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_status)) {
	crm_trace("Reusing previously calculated cluster status");
	pcmk__set_scheduler_flags(scheduler, flags);
	return;
	}

	if (scheduler->localhost) {
	localhost_save = scheduler->localhost;
	}

	CRM_ASSERT(cib != NULL);
	crm_trace("Calculating cluster status");

	/* This will zero the entire struct without freeing anything first, so
	* callers should never call pcmk__schedule_actions() with a populated data
	* set unless pcmk_sched_have_status is set (i.e. cluster_status() was
	* previously called, whether directly or via pcmk__schedule_actions()).
	*/
	set_working_set_defaults(scheduler);

	if (localhost_save) {
	scheduler->localhost = localhost_save;
	}

	pcmk__set_scheduler_flags(scheduler, flags);
	scheduler->input = cib;
	cluster_status(scheduler); // Sets pcmk_sched_have_status
	}

	/*!
	* \internal
	* \brief Run the scheduler for a given CIB
	*
	* \param[in,out] cib CIB XML to use as scheduler input
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
	pcmk_scheduler_t *scheduler)
	{
	unpack_cib(cib, flags, scheduler);
	pcmk__set_assignment_methods(scheduler);
	pcmk__apply_node_health(scheduler);
	pcmk__unpack_constraints(scheduler);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_validate_only)) {
	return;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)
	&& pcmk__is_daemon) {
	log_resource_details(scheduler);
	}

	apply_node_criteria(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	return;
	}

	pcmk__create_internal_constraints(scheduler);
	pcmk__handle_rsc_config_changes(scheduler);
	assign_resources(scheduler);
	schedule_resource_actions(scheduler);

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(scheduler);

	schedule_fencing_and_shutdowns(scheduler);
	pcmk__apply_orderings(scheduler);
	log_all_actions(scheduler);
	pcmk__create_graph(scheduler);

	if (get_crm_log_level() == LOG_TRACE) {
	log_unrunnable_actions(scheduler);
	}
	}

	/*!
	* \internal
	* \brief Initialize scheduler data
	*
	* Make our own copies of the CIB XML and date/time object, if they're not
	* \c NULL. This way we don't have to take ownership of the objects passed via
	* the API.
	*
	* This function is most useful for public API functions that want the caller
	* to retain ownership of the CIB object
	*
	* \param[in,out] out Output object
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Date and time to use in the scheduler (if \c NULL,
	* use current date and time). This can be used for
	* checking whether a rule is in effect at a certa
	* date and time.
	* \param[out] scheduler Where to store initialized scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__init_scheduler(pcmk__output_t out, xmlNodePtr input, const crm_time_t date,
	pcmk_scheduler_t **scheduler)
	{
	// Allows for cleaner syntax than dereferencing the scheduler argument
	pcmk_scheduler_t *new_scheduler = NULL;

	new_scheduler = pe_new_working_set();
	if (new_scheduler == NULL) {
	return ENOMEM;
	}

	pcmk__set_scheduler_flags(new_scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);

	// Populate the scheduler data

	// Make our own copy of the given input or fetch the CIB and use that
	if (input != NULL) {
	new_scheduler->input = pcmk__xml_copy(NULL, input);
	if (new_scheduler->input == NULL) {
	out->err(out, "Failed to copy input XML");
	pe_free_working_set(new_scheduler);
	return ENOMEM;
	}

	} else {
	int rc = cib__signon_query(out, NULL, &(new_scheduler->input));

	if (rc != pcmk_rc_ok) {
	pe_free_working_set(new_scheduler);
	return rc;
	}
	}

	// Make our own copy of the given crm_time_t object; otherwise
	// cluster_status() populates with the current time
	if (date != NULL) {
	// pcmk_copy_time() guarantees non-NULL
	new_scheduler->now = pcmk_copy_time(date);
	}

	// Unpack everything
	cluster_status(new_scheduler);
	*scheduler = new_scheduler;

	return pcmk_rc_ok;
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 8cf8174c63..003a9ebfab 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2260 +1,2266 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pcmk__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
	"requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
	data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network,
	PCMK_VALUE_HOST, pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -PCMK_SCORE_INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = pcmk__xe_id(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
	PCMK_VALUE_MINUS_INFINITY, parent->cluster);
	} else {
	node->weight = -PCMK_SCORE_INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -PCMK_SCORE_INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pcmk__is_pacemaker_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -PCMK_SCORE_INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pcmk__is_guest_or_bundle_node() functions correctly
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	pcmk__insert_dup(replica->node->details->attrs,
	CRM_ATTR_KIND, "container");

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	pcmk__insert_meta(replica->child, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
	sizeof(pe__bundle_mount_t));

	mount->source = pcmk__str_copy(source);
	mount->target = pcmk__str_copy(target);
	mount->options = pcmk__str_copy(options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, xmlNode xml,
	const char *field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pcmk__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pcmk__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", pcmk__xe_id(mount_xml), \
	flags, (flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	const xmlNode *xml_child = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_DOCKER, NULL, NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK__XE_RKT, NULL, NULL);
	if (xml_obj != NULL) {
	pcmk__warn_once(pcmk__wo_rkt,
	"Support for " PCMK__XE_RKT " in bundles "
	"(such as %s) is deprecated and will be "
	"removed in a future release", rsc->id);
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PODMAN, NULL,
	NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, PCMK__XA_PROMOTED_MAX_LEGACY);
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command =
	crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
	bundle_data->launcher_options = crm_element_value_copy(xml_obj,
	PCMK_XA_OPTIONS);
	bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
	bundle_data->container_network = crm_element_value_copy(xml_obj,
	PCMK_XA_NETWORK);

	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_NETWORK, NULL, NULL);
	if(xml_obj) {
	bundle_data->ip_range_start =
	crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
	bundle_data->host_netmask =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
	bundle_data->host_network =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
	bundle_data->control_port =
	crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
	value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	pe__bundle_port_t *port =
	pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));

	port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
	} else {
	port->target = crm_element_value_copy(xml_child,
	PCMK_XA_INTERNAL_PORT);
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
	pcmk__xe_id(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_STORAGE, NULL, NULL);
	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
	const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
	const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s",
	pcmk__xe_id(xml_child));
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PRIMITIVE, NULL, NULL);
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	pcmk__xml_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
	rsc->id, pcmk__xe_id(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting
	* PCMK_XA_CONTROL_PORT.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	pcmk__insert_dup(rsc->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	pcmk__insert_dup(bundle_data->child->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (pcmk__same_node(replica->node, node)) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_managed));
	status_print("failed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " PCMK_XA_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	const char *type = container_agent_str(bundle_data->agent_type);
	const char *unique = pcmk__flag_text(rsc->flags, pcmk_rsc_unique);
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk_rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
	const char *failed = pcmk__flag_text(rsc->flags, pcmk_rsc_failed);

	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE, 8,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, type,
	PCMK_XA_IMAGE, bundle_data->image,
	PCMK_XA_UNIQUE, unique,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_FAILED, failed,
	PCMK_XA_DESCRIPTION, desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA, 1,
	PCMK_XA_ID, id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	- out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	+ out->message(out, (const char *) replica->ip->xml->name, show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	- out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	- replica->child, only_node, only_rsc);
	+ out->message(out, (const char *) replica->child->xml->name,
	+ show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	- out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	- replica->container, only_node, only_rsc);
	+ out->message(out, (const char *) replica->container->xml->name,
	+ show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	- out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	- replica->remote, only_node, only_rsc);
	+ out->message(out, (const char *) replica->remote->xml->name,
	+ show_opts, replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	- out->message(out, crm_map_element_name(replica->ip->xml),
	+ out->message(out, (const char *) replica->ip->xml->name,
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	- out->message(out, crm_map_element_name(replica->child->xml),
	- new_show_opts, replica->child, only_node, only_rsc);
	+ out->message(out, (const char *) replica->child->xml->name,
	+ new_show_opts, replica->child, only_node,
	+ only_rsc);
	}

	if (print_ctnr) {
	- out->message(out, crm_map_element_name(replica->container->xml),
	- new_show_opts, replica->container, only_node, only_rsc);
	+ out->message(out, (const char *) replica->container->xml->name,
	+ new_show_opts, replica->container, only_node,
	+ only_rsc);
	}

	if (print_remote) {
	- out->message(out, crm_map_element_name(replica->remote->xml),
	- new_show_opts, replica->remote, only_node, only_rsc);
	+ out->message(out, (const char *) replica->remote->xml->name,
	+ new_show_opts, replica->remote, only_node,
	+ only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica,
	pcmk__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	- out->message(out, crm_map_element_name(replica->ip->xml),
	+ out->message(out, (const char *) replica->ip->xml->name,
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	- out->message(out, crm_map_element_name(replica->child->xml),
	- new_show_opts, replica->child, only_node, only_rsc);
	+ out->message(out, (const char *) replica->child->xml->name,
	+ new_show_opts, replica->child, only_node,
	+ only_rsc);
	}

	if (print_ctnr) {
	- out->message(out, crm_map_element_name(replica->container->xml),
	- new_show_opts, replica->container, only_node, only_rsc);
	+ out->message(out, (const char *) replica->container->xml->name,
	+ new_show_opts, replica->container, only_node,
	+ only_rsc);
	}

	if (print_remote) {
	- out->message(out, crm_map_element_name(replica->remote->xml),
	- new_show_opts, replica->remote, only_node, only_rsc);
	+ out->message(out, (const char *) replica->remote->xml->name,
	+ new_show_opts, replica->remote, only_node,
	+ only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica,
	pcmk__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pcmk__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pcmk_node_t *node = NULL;
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pcmk__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk_rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
	index c4e9ab394e..2820bb04e0 100644
	--- a/lib/pengine/clone.c
	+++ b/lib/pengine/clone.c
	@@ -1,1537 +1,1537 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>
	#include <crm/common/xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED_LEGACY "s"
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED_LEGACY "s"
	#else
	#define PROMOTED_INSTANCES PCMK__ROLE_PROMOTED
	#define UNPROMOTED_INSTANCES PCMK__ROLE_UNPROMOTED
	#endif

	typedef struct clone_variant_data_s {
	int clone_max;
	int clone_node_max;

	int promoted_max;
	int promoted_node_max;

	int total_clones;

	uint32_t flags; // Group of enum pcmk__clone_flags

	notify_data_t *stop_notify;
	notify_data_t *start_notify;
	notify_data_t *demote_notify;
	notify_data_t *promote_notify;

	xmlNode *xml_obj_child;
	} clone_variant_data_t;

	#define get_clone_variant_data(data, rsc) \
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_clone)); \
	data = (clone_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be run
	*
	* \param[in] clone Clone or clone instance to check
	*
	* \return Maximum instances for \p clone
	*/
	int
	pe__clone_max(const pcmk_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed per node
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum allowed instances per node for \p clone
	*/
	int
	pe__clone_node_max(const pcmk_resource_t *clone)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->clone_node_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_max(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_max;
	}

	/*!
	* \internal
	* \brief Return the maximum number of clone instances allowed to be promoted
	*
	* \param[in] clone Promotable clone or clone instance to check
	*
	* \return Maximum promoted instances for \p clone
	*/
	int
	pe__clone_promoted_node_max(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
	return clone_data->promoted_node_max;
	}

	static GList *
	sorted_hash_table_values(GHashTable *table)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
	retval = g_list_prepend(retval, (char *) value);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) strcmp);
	return retval;
	}

	static GList *
	nodes_with_status(GHashTable table, const char status)
	{
	GList *retval = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, table);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	if (!strcmp((char *) value, status)) {
	retval = g_list_prepend(retval, key);
	}
	}

	retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
	return retval;
	}

	static GString *
	node_list_to_str(const GList *list)
	{
	GString *retval = NULL;

	for (const GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__add_word(&retval, 1024, (const char *) iter->data);
	}

	return retval;
	}

	static void
	clone_header(pcmk__output_t out, int rc, const pcmk_resource_t *rsc,
	clone_variant_data_t clone_data, const char desc)
	{
	GString *attrs = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	pcmk__add_separated_word(&attrs, 64, "promotable", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	pcmk__add_separated_word(&attrs, 64, "unique", ", ");
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
	}

	if (attrs != NULL) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s] (%s)%s%s%s",
	rsc->id,
	pcmk__xe_id(clone_data->xml_obj_child),
	(const char *) attrs->str, desc ? " (" : "",
	desc ? desc : "", desc ? ")" : "");
	g_string_free(attrs, TRUE);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s]%s%s%s",
	rsc->id,
	pcmk__xe_id(clone_data->xml_obj_child),
	desc ? " (" : "", desc ? desc : "",
	desc ? ")" : "");
	}
	}

	void
	pe__force_anon(const char standard, pcmk_resource_t rsc, const char *rid,
	pcmk_scheduler_t *scheduler)
	{
	if (pcmk__is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	pcmk__config_warn("Ignoring " PCMK_META_GLOBALLY_UNIQUE " for %s "
	"because %s resources such as %s can be used only as "
	"anonymous clones", rsc->id, standard, rid);

	clone_data->clone_node_max = 1;
	clone_data->clone_max = QB_MIN(clone_data->clone_max,
	g_list_length(scheduler->nodes));
	}
	}

	pcmk_resource_t *
	find_clone_instance(const pcmk_resource_t rsc, const char sub_id)
	{
	char *child_id = NULL;
	pcmk_resource_t *child = NULL;
	const char *child_base = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	child_base = pcmk__xe_id(clone_data->xml_obj_child);
	child_id = crm_strdup_printf("%s:%s", child_base, sub_id);
	child = pe_find_resource(rsc->children, child_id);

	free(child_id);
	return child;
	}

	pcmk_resource_t *
	pe__create_clone_child(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	gboolean as_orphan = FALSE;
	char *inc_num = NULL;
	char *inc_max = NULL;
	pcmk_resource_t *child_rsc = NULL;
	xmlNode *child_copy = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);

	if (clone_data->total_clones >= clone_data->clone_max) {
	// If we've already used all available instances, this is an orphan
	as_orphan = TRUE;
	}

	// Allocate instance numbers in numerical order (starting at 0)
	inc_num = pcmk__itoa(clone_data->total_clones);
	inc_max = pcmk__itoa(clone_data->clone_max);

	child_copy = pcmk__xml_copy(NULL, clone_data->xml_obj_child);

	crm_xml_add(child_copy, PCMK__META_CLONE, inc_num);

	if (pe__unpack_resource(child_copy, &child_rsc, rsc,
	scheduler) != pcmk_rc_ok) {
	goto bail;
	}
	/* child_rsc->globally_unique = rsc->globally_unique; */

	CRM_ASSERT(child_rsc);
	clone_data->total_clones += 1;
	pcmk__rsc_trace(child_rsc, "Setting clone attributes for: %s",
	child_rsc->id);
	rsc->children = g_list_append(rsc->children, child_rsc);
	if (as_orphan) {
	pe__set_resource_flags_recursive(child_rsc, pcmk_rsc_removed);
	}

	pcmk__insert_meta(child_rsc, PCMK_META_CLONE_MAX, inc_max);
	pcmk__rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);

	bail:
	free(inc_num);
	free(inc_max);

	return child_rsc;
	}

	/*!
	* \internal
	* \brief Unpack a nonnegative integer value from a resource meta-attribute
	*
	* \param[in] rsc Resource with meta-attribute
	* \param[in] meta_name Name of meta-attribute to unpack
	* \param[in] deprecated_name If not NULL, try unpacking this
	* if \p meta_name is unset
	* \param[in] default_value Value to use if unset
	*
	* \return Integer parsed from resource's specified meta-attribute if a valid
	* nonnegative integer, \p default_value if unset, or 0 if invalid
	*/
	static int
	unpack_meta_int(const pcmk_resource_t rsc, const char meta_name,
	const char *deprecated_name, int default_value)
	{
	int integer = default_value;
	const char *value = g_hash_table_lookup(rsc->meta, meta_name);

	if ((value == NULL) && (deprecated_name != NULL)) {
	value = g_hash_table_lookup(rsc->meta, deprecated_name);
	}
	if (value != NULL) {
	pcmk__scan_min_int(value, &integer, 0);
	}
	return integer;
	}

	gboolean
	clone_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	int lpc = 0;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = rsc->xml;
	clone_variant_data_t *clone_data = NULL;

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	clone_data = pcmk__assert_alloc(1, sizeof(clone_variant_data_t));
	rsc->variant_opaque = clone_data;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	// Use 1 as default but 0 for minimum and invalid
	// @COMPAT PCMK__META_PROMOTED_MAX_LEGACY deprecated since 2.0.0
	clone_data->promoted_max =
	unpack_meta_int(rsc, PCMK_META_PROMOTED_MAX,
	PCMK__META_PROMOTED_MAX_LEGACY, 1);

	// Use 1 as default but 0 for minimum and invalid
	// @COMPAT PCMK__META_PROMOTED_NODE_MAX_LEGACY deprecated since 2.0.0
	clone_data->promoted_node_max =
	unpack_meta_int(rsc, PCMK_META_PROMOTED_NODE_MAX,
	PCMK__META_PROMOTED_NODE_MAX_LEGACY, 1);
	}

	// Use 1 as default but 0 for minimum and invalid
	clone_data->clone_node_max = unpack_meta_int(rsc, PCMK_META_CLONE_NODE_MAX,
	NULL, 1);

	/* Use number of nodes (but always at least 1, which is handy for crm_verify
	* for a CIB without nodes) as default, but 0 for minimum and invalid
	*/
	clone_data->clone_max = unpack_meta_int(rsc, PCMK_META_CLONE_MAX, NULL,
	QB_MAX(1, g_list_length(scheduler->nodes)));

	if (crm_is_true(g_hash_table_lookup(rsc->meta, PCMK_META_ORDERED))) {
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", rsc->id,
	clone_data->flags,
	pcmk__clone_ordered,
	"pcmk__clone_ordered");
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_node_max > 1)) {

	pcmk__config_err("Ignoring " PCMK_META_CLONE_NODE_MAX " of %d for %s "
	"because anonymous clones support only one instance "
	"per node", clone_data->clone_node_max, rsc->id);
	clone_data->clone_node_max = 1;
	}

	pcmk__rsc_trace(rsc, "Options for %s", rsc->id);
	pcmk__rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
	pcmk__rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
	pcmk__rsc_trace(rsc, "\tClone is unique: %s",
	pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
	pcmk__rsc_trace(rsc, "\tClone is promotable: %s",
	pcmk__flag_text(rsc->flags, pcmk_rsc_promotable));

	// Clones may contain a single group or primitive
	for (a_child = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
	a_child != NULL; a_child = pcmk__xe_next(a_child)) {

	if (pcmk__str_any_of((const char *) a_child->name,
	PCMK_XE_PRIMITIVE, PCMK_XE_GROUP, NULL)) {
	clone_data->xml_obj_child = a_child;
	break;
	}
	}

	if (clone_data->xml_obj_child == NULL) {
	pcmk__config_err("%s has nothing to clone", rsc->id);
	return FALSE;
	}

	/*
	* Make clones ever so slightly sticky by default
	*
	* This helps ensure clone instances are not shuffled around the cluster
	* for no benefit in situations when pre-allocation is not appropriate
	*/
	if (g_hash_table_lookup(rsc->meta, PCMK_META_RESOURCE_STICKINESS) == NULL) {
	pcmk__insert_meta(rsc, PCMK_META_RESOURCE_STICKINESS, "1");
	}

	/* This ensures that the PCMK_META_GLOBALLY_UNIQUE value always exists for
	* children to inherit when being unpacked, as well as in resource agents'
	* environment.
	*/
	pcmk__insert_meta(rsc, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__flag_text(rsc->flags, pcmk_rsc_unique));

	if (clone_data->clone_max <= 0) {
	/* Create one child instance so that unpack_find_resource() will hook up
	* any orphans up to the parent correctly.
	*/
	if (pe__create_clone_child(rsc, scheduler) == NULL) {
	return FALSE;
	}

	} else {
	// Create a child instance for each available instance number
	for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
	if (pe__create_clone_child(rsc, scheduler) == NULL) {
	return FALSE;
	}
	}
	}

	pcmk__rsc_trace(rsc, "Added %d children to resource %s...",
	clone_data->clone_max, rsc->id);
	return TRUE;
	}

	gboolean
	clone_active(pcmk_resource_t * rsc, gboolean all)
	{
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean child_active = child_rsc->fns->active(child_rsc, all);

	if (all == FALSE && child_active) {
	return TRUE;
	} else if (all && child_active == FALSE) {
	return FALSE;
	}
	}

	if (all) {
	return TRUE;
	} else {
	return FALSE;
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	short_print(const char list, const char prefix, const char *type,
	const char suffix, long options, void print_data)
	{
	if(suffix == NULL) {
	suffix = "";
	}

	if (!pcmk__str_empty(list)) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	status_print("%s%s: [ %s ]%s", prefix, type, list, suffix);

	if (options & pe_print_html) {
	status_print("</li>\n");

	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}

	}
	}

	static const char *
	configured_role_str(pcmk_resource_t * rsc)
	{
	const char *target_role = g_hash_table_lookup(rsc->meta,
	PCMK_META_TARGET_ROLE);

	if ((target_role == NULL) && rsc->children && rsc->children->data) {
	pcmk_resource_t *instance = rsc->children->data; // Any instance will do

	target_role = g_hash_table_lookup(instance->meta,
	PCMK_META_TARGET_ROLE);
	}
	return target_role;
	}

	static enum rsc_role_e
	configured_role(pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = pcmk_role_unknown;
	const char *target_role = configured_role_str(rsc);

	if (target_role != NULL) {
	role = pcmk_parse_role(target_role);
	if (role == pcmk_role_unknown) {
	pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
	" for resource %s", rsc->id);
	}
	}
	return role;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	clone_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	char *child_text = crm_strdup_printf("%s ", pre_text);
	const char *target_role = configured_role_str(rsc);
	GList *gIter = rsc->children;

	status_print("%s<clone ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
	status_print("multi_state=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_promotable));
	status_print("unique=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_managed));
	status_print("failed=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pcmk__flag_text(rsc->flags, pcmk_rsc_ignore_failure));
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</clone>\n", pre_text);
	free(child_text);
	}

	bool
	is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any)
	{
	GList *gIter;
	bool all = !any;

	if (pcmk_is_set(rsc->flags, flag)) {
	if(any) {
	return TRUE;
	}
	} else if(all) {
	return FALSE;
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	if(is_set_recursive(gIter->data, flag, any)) {
	if(any) {
	return TRUE;
	}

	} else if(all) {
	return FALSE;
	}
	}

	if(all) {
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	clone_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	GString *list_text = NULL;
	char *child_text = NULL;
	GString *stopped_list = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	clone_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_clone_variant_data(clone_data, rsc);

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sClone Set: %s [%s]%s%s%s",
	pcmk__s(pre_text, ""), rsc->id,
	pcmk__xe_id(clone_data->xml_obj_child),
	pcmk_is_set(rsc->flags, pcmk_rsc_promotable)? " (promotable)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (options & pe_print_clone_details) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(options, pe_print_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(options, pe_print_clone_active)) {

	pcmk__add_word(&stopped_list, 1024, child_rsc->id);
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pcmk_node_t *location = NULL;

	location = child_rsc->fns->location(child_rsc, NULL, TRUE);
	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	short_print((const char *) list_text->str, child_text,
	PROMOTED_INSTANCES, NULL, options, print_data);
	g_string_truncate(list_text, 0);
	}
	g_list_free(promoted_list);

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}

	if (list_text != NULL) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES " (" PCMK_META_TARGET_ROLE ")",
	NULL, options, print_data);
	} else {
	short_print((const char *) list_text->str, child_text,
	UNPROMOTED_INSTANCES, NULL, options, print_data);
	}

	} else {
	short_print((const char *) list_text->str, child_text, "Started",
	NULL, options, print_data);
	}
	}

	g_list_free(started_list);

	if (!pcmk_is_set(options, pe_print_clone_active)) {
	const char *state = "Stopped";
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped list for non-unique clones */
	if (stopped_list != NULL) {
	g_string_truncate(stopped_list, 0);
	}

	if (list == NULL) {
	/* Clusters with PCMK_OPT_SYMMETRIC_CLUSTER=false haven't
	* calculated allowed_nodes yet. If we've not probed for them
	* yet, the Stopped list will be empty.
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pcmk_node_t node = (pcmk_node_t ) nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL) {
	pcmk__add_word(&stopped_list, 1024, node->details->uname);
	}
	}
	g_list_free(list);
	}

	if (stopped_list != NULL) {
	short_print((const char *) stopped_list->str, child_text, state,
	NULL, options, print_data);
	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (stopped_list != NULL) {
	g_string_free(stopped_list, TRUE);
	}
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__clone_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GList *gIter = rsc->children;
	GList *all = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	all = g_list_prepend(all, (gpointer) "*");

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (!printed_header) {
	const char *multi_state = pcmk__flag_text(rsc->flags,
	pcmk_rsc_promotable);
	const char *unique = pcmk__flag_text(rsc->flags, pcmk_rsc_unique);
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk_rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
	const char *disabled = pcmk__btoa(pe__resource_is_disabled(rsc));
	const char *failed = pcmk__flag_text(rsc->flags, pcmk_rsc_failed);
	const char *ignored = pcmk__flag_text(rsc->flags,
	pcmk_rsc_ignore_failure);
	const char *target_role = configured_role_str(rsc);
	const char *desc = pe__resource_description(rsc, show_opts);

	printed_header = TRUE;

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_CLONE, 10,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_MULTI_STATE, multi_state,
	PCMK_XA_UNIQUE, unique,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_DISABLED, disabled,
	PCMK_XA_FAILED, failed,
	PCMK_XA_FAILURE_IGNORED, ignored,
	PCMK_XA_TARGET_ROLE, target_role,
	PCMK_XA_DESCRIPTION, desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	- out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	+ out->message(out, (const char *) child_rsc->xml->name, show_opts,
	child_rsc, only_node, all);
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out);
	}

	g_list_free(all);
	return rc;
	}

	PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__clone_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	GHashTable *stopped = NULL;

	GString *list_text = NULL;

	GList *promoted_list = NULL;
	GList *started_list = NULL;
	GList *gIter = rsc->children;

	const char *desc = NULL;

	clone_variant_data_t *clone_data = NULL;
	int active_instances = 0;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_clone_variant_data(clone_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean print_full = FALSE;
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	gboolean partially_active = child_rsc->fns->active(child_rsc, FALSE);

	if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
	continue;
	}

	if (child_rsc->fns->is_filtered(child_rsc, only_rsc, print_everything)) {
	continue;
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	print_full = TRUE;
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	// Print individual instance when unique (except stopped orphans)
	if (partially_active
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	print_full = TRUE;
	}

	// Everything else in this block is for anonymous clones

	} else if (pcmk_is_set(show_opts, pcmk_show_pending)
	&& (child_rsc->pending_task != NULL)
	&& strcmp(child_rsc->pending_task, "probe")) {
	// Print individual instance when non-probe action is pending
	print_full = TRUE;

	} else if (partially_active == FALSE) {
	// List stopped instances when requested (except orphans)
	if (!pcmk_is_set(child_rsc->flags, pcmk_rsc_removed)
	&& !pcmk_is_set(show_opts, pcmk_show_clone_detail)
	&& pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	pcmk__insert_dup(stopped, child_rsc->id, "Stopped");
	}

	} else if (is_set_recursive(child_rsc, pcmk_rsc_removed, TRUE)
	\|\| !is_set_recursive(child_rsc, pcmk_rsc_managed, FALSE)
	\|\| is_set_recursive(child_rsc, pcmk_rsc_failed, TRUE)) {

	// Print individual instance when active orphaned/unmanaged/failed
	print_full = TRUE;

	} else if (child_rsc->fns->active(child_rsc, TRUE)) {
	// Instance of fully active anonymous clone

	pcmk_node_t *location = NULL;

	location = child_rsc->fns->location(child_rsc, NULL, TRUE);
	if (location) {
	// Instance is active on a single node

	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, TRUE);

	if (location->details->online == FALSE && location->details->unclean) {
	print_full = TRUE;

	} else if (a_role > pcmk_role_unpromoted) {
	promoted_list = g_list_append(promoted_list, location);

	} else {
	started_list = g_list_append(started_list, location);
	}

	} else {
	/* uncolocated group - bleh */
	print_full = TRUE;
	}

	} else {
	// Instance of partially active anonymous clone
	print_full = TRUE;
	}

	if (print_full) {
	GList *all = NULL;

	clone_header(out, &rc, rsc, clone_data, desc);

	/* Print every resource that's a child of this clone. */
	all = g_list_prepend(all, (gpointer) "*");
	- out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	+ out->message(out, (const char *) child_rsc->xml->name, show_opts,
	child_rsc, only_node, all);
	g_list_free(all);
	}
	}

	if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return pcmk_rc_ok;
	}

	/* Promoted */
	promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
	for (gIter = promoted_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(promoted_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	out->list_item(out, NULL, PROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	g_string_truncate(list_text, 0);
	}

	/* Started/Unpromoted */
	started_list = g_list_sort(started_list, pe__cmp_node_name);
	for (gIter = started_list; gIter; gIter = gIter->next) {
	pcmk_node_t *host = gIter->data;

	if (!pcmk__str_in_list(host->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	pcmk__add_word(&list_text, 1024, host->details->uname);
	active_instances++;
	}
	g_list_free(started_list);

	if ((list_text != NULL) && (list_text->len > 0)) {
	clone_header(out, &rc, rsc, clone_data, desc);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	enum rsc_role_e role = configured_role(rsc);

	if (role == pcmk_role_unpromoted) {
	out->list_item(out, NULL,
	UNPROMOTED_INSTANCES
	" (" PCMK_META_TARGET_ROLE "): [ %s ]",
	(const char *) list_text->str);
	} else {
	out->list_item(out, NULL, UNPROMOTED_INSTANCES ": [ %s ]",
	(const char *) list_text->str);
	}

	} else {
	out->list_item(out, NULL, "Started: [ %s ]",
	(const char *) list_text->str);
	}
	}

	if (list_text != NULL) {
	g_string_free(list_text, TRUE);
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& (clone_data->clone_max > active_instances)) {

	GList *nIter;
	GList *list = g_hash_table_get_values(rsc->allowed_nodes);

	/* Custom stopped table for non-unique clones */
	if (stopped != NULL) {
	g_hash_table_destroy(stopped);
	stopped = NULL;
	}

	if (list == NULL) {
	/* Clusters with PCMK_OPT_SYMMETRIC_CLUSTER=false haven't
	* calculated allowed_nodes yet. If we've not probed for them
	* yet, the Stopped list will be empty.
	*/
	list = g_hash_table_get_values(rsc->known_on);
	}

	list = g_list_sort(list, pe__cmp_node_name);
	for (nIter = list; nIter != NULL; nIter = nIter->next) {
	pcmk_node_t node = (pcmk_node_t ) nIter->data;

	if (pe_find_node(rsc->running_on, node->details->uname) == NULL &&
	pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node->details->uname);
	const char *state = "Stopped";

	if (configured_role(rsc) == pcmk_role_stopped) {
	state = "Stopped (disabled)";
	}

	if (stopped == NULL) {
	stopped = pcmk__strkey_table(free, free);
	}
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op,
	PCMK__XA_RC_CODE),
	&rc, 0);
	g_hash_table_insert(stopped, strdup(node->details->uname),
	crm_strdup_printf("Stopped (%s)", services_ocf_exitcode_str(rc)));
	} else {
	pcmk__insert_dup(stopped, node->details->uname, state);
	}
	}
	}
	g_list_free(list);
	}

	if (stopped != NULL) {
	GList *list = sorted_hash_table_values(stopped);

	clone_header(out, &rc, rsc, clone_data, desc);

	for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
	const char *status = status_iter->data;
	GList *nodes = nodes_with_status(stopped, status);
	GString *nodes_str = node_list_to_str(nodes);

	if (nodes_str != NULL) {
	if (nodes_str->len > 0) {
	out->list_item(out, NULL, "%s: [ %s ]", status,
	(const char *) nodes_str->str);
	}
	g_string_free(nodes_str, TRUE);
	}

	g_list_free(nodes);
	}

	g_list_free(list);
	g_hash_table_destroy(stopped);

	/* If there are no instances of this clone (perhaps because there are no
	* nodes configured), simply output the clone header by itself. This can
	* come up in PCS testing.
	*/
	} else if (active_instances == 0) {
	clone_header(out, &rc, rsc, clone_data, desc);
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	void
	clone_free(pcmk_resource_t * rsc)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pcmk__rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	free_xml(child_rsc->xml);
	child_rsc->xml = NULL;
	/* There could be a saved unexpanded xml */
	free_xml(child_rsc->orig_xml);
	child_rsc->orig_xml = NULL;
	child_rsc->fns->free(child_rsc);
	}

	g_list_free(rsc->children);

	if (clone_data) {
	CRM_ASSERT(clone_data->demote_notify == NULL);
	CRM_ASSERT(clone_data->stop_notify == NULL);
	CRM_ASSERT(clone_data->start_notify == NULL);
	CRM_ASSERT(clone_data->promote_notify == NULL);
	}

	common_free(rsc);
	}

	enum rsc_role_e
	clone_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e clone_role = pcmk_role_unknown;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	enum rsc_role_e a_role = child_rsc->fns->state(child_rsc, current);

	if (a_role > clone_role) {
	clone_role = a_role;
	}
	}

	pcmk__rsc_trace(rsc, "%s role: %s", rsc->id, pcmk_role_text(clone_role));
	return clone_role;
	}

	/*!
	* \internal
	* \brief Check whether a clone has an instance for every node
	*
	* \param[in] rsc Clone to check
	* \param[in] scheduler Scheduler data
	*/
	bool
	pe__is_universal_clone(const pcmk_resource_t *rsc,
	const pcmk_scheduler_t *scheduler)
	{
	if (pcmk__is_clone(rsc)) {
	clone_variant_data_t *clone_data = rsc->variant_opaque;

	if (clone_data->clone_max == g_list_length(scheduler->nodes)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gboolean
	pe__clone_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	clone_variant_data_t *clone_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_clone_variant_data(clone_data, rsc);
	passes = pcmk__str_in_list(pcmk__xe_id(clone_data->xml_obj_child),
	only_rsc, pcmk__str_star_matches);

	if (!passes) {
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t *child_rsc = NULL;

	child_rsc = (const pcmk_resource_t *) iter->data;
	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}
	}
	return !passes;
	}

	const char *
	pe__clone_child_id(const pcmk_resource_t *rsc)
	{
	clone_variant_data_t *clone_data = NULL;
	get_clone_variant_data(clone_data, rsc);
	return pcmk__xe_id(clone_data->xml_obj_child);
	}

	/*!
	* \internal
	* \brief Check whether a clone is ordered
	*
	* \param[in] clone Clone resource to check
	*
	* \return true if clone is ordered, otherwise false
	*/
	bool
	pe__clone_is_ordered(const pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	return pcmk_is_set(clone_data->flags, pcmk__clone_ordered);
	}

	/*!
	* \internal
	* \brief Set a clone flag
	*
	* \param[in,out] clone Clone resource to set flag for
	* \param[in] flag Clone flag to set
	*
	* \return Standard Pacemaker return code (either pcmk_rc_ok if flag was not
	* already set or pcmk_rc_already if it was)
	*/
	int
	pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	if (pcmk_is_set(clone_data->flags, flag)) {
	return pcmk_rc_already;
	}
	clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
	"Clone", clone->id,
	clone_data->flags, flag, "flag");
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a clone flag is set
	*
	* \param[in] group Clone resource to check
	* \param[in] flags Flag or flags to check
	*
	* \return \c true if all \p flags are set for \p clone, otherwise \c false
	*/
	bool
	pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);
	CRM_ASSERT(clone_data != NULL);

	return pcmk_all_flags_set(clone_data->flags, flags);
	}

	/*!
	* \internal
	* \brief Create pseudo-actions needed for promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[in] any_promoting Whether any instances will be promoted
	* \param[in] any_demoting Whether any instance will be demoted
	*/
	void
	pe__create_promotable_pseudo_ops(pcmk_resource_t *clone, bool any_promoting,
	bool any_demoting)
	{
	pcmk_action_t *action = NULL;
	pcmk_action_t *action_complete = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	// Create a "promote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTE,
	!any_promoting, true);

	// Create a "promoted" action for when all promotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTED,
	!any_promoting, true);
	action_complete->priority = PCMK_SCORE_INFINITY;

	// Create notification pseudo-actions for promotion
	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_PROMOTE,
	action,
	action_complete);
	}

	// Create a "demote" action for the clone itself
	action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTE,
	!any_demoting, true);

	// Create a "demoted" action for when all demotions are done
	action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTED,
	!any_demoting, true);
	action_complete->priority = PCMK_SCORE_INFINITY;

	// Create notification pseudo-actions for demotion
	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_DEMOTE,
	action,
	action_complete);

	if (clone_data->promote_notify != NULL) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->start_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->promote_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->start_notify->pre, pcmk__ar_ordered);
	order_actions(clone_data->demote_notify->post_done,
	clone_data->stop_notify->pre, pcmk__ar_ordered);
	}
	}
	}

	/*!
	* \internal
	* \brief Create all notification data and actions for a clone
	*
	* \param[in,out] clone Clone to create notifications for
	*/
	void
	pe__create_clone_notifications(pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__create_action_notifications(clone, clone_data->start_notify);
	pe__create_action_notifications(clone, clone_data->stop_notify);
	pe__create_action_notifications(clone, clone_data->promote_notify);
	pe__create_action_notifications(clone, clone_data->demote_notify);
	}

	/*!
	* \internal
	* \brief Free all notification data for a clone
	*
	* \param[in,out] clone Clone to free notification data for
	*/
	void
	pe__free_clone_notification_data(pcmk_resource_t *clone)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	pe__free_action_notification_data(clone_data->demote_notify);
	clone_data->demote_notify = NULL;

	pe__free_action_notification_data(clone_data->stop_notify);
	clone_data->stop_notify = NULL;

	pe__free_action_notification_data(clone_data->start_notify);
	clone_data->start_notify = NULL;

	pe__free_action_notification_data(clone_data->promote_notify);
	clone_data->promote_notify = NULL;
	}

	/*!
	* \internal
	* \brief Create pseudo-actions for clone start/stop notifications
	*
	* \param[in,out] clone Clone to create pseudo-actions for
	* \param[in,out] start Start action for \p clone
	* \param[in,out] stop Stop action for \p clone
	* \param[in,out] started Started action for \p clone
	* \param[in,out] stopped Stopped action for \p clone
	*/
	void
	pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
	pcmk_action_t start, pcmk_action_t started,
	pcmk_action_t stop, pcmk_action_t stopped)
	{
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, clone);

	if (clone_data->start_notify == NULL) {
	clone_data->start_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_START,
	start, started);
	}

	if (clone_data->stop_notify == NULL) {
	clone_data->stop_notify = pe__action_notif_pseudo_ops(clone,
	PCMK_ACTION_STOP,
	stop, stopped);
	if ((clone_data->start_notify != NULL)
	&& (clone_data->stop_notify != NULL)) {
	order_actions(clone_data->stop_notify->post_done,
	clone_data->start_notify->pre, pcmk__ar_ordered);
	}
	}
	}

	/*!
	* \internal
	* \brief Get maximum clone resource instances per node
	*
	* \param[in] rsc Clone resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__clone_max_per_node(const pcmk_resource_t *rsc)
	{
	const clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);
	return clone_data->clone_node_max;
	}
	diff --git a/lib/pengine/group.c b/lib/pengine/group.c
	index 906eb6a7a7..314e2b233b 100644
	--- a/lib/pengine/group.c
	+++ b/lib/pengine/group.c
	@@ -1,540 +1,540 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/output.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	typedef struct group_variant_data_s {
	pcmk_resource_t *last_child; // Last group member
	uint32_t flags; // Group of enum pcmk__group_flags
	} group_variant_data_t;

	/*!
	* \internal
	* \brief Get a group's last member
	*
	* \param[in] group Group resource to check
	*
	* \return Last member of \p group if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__last_group_member(const pcmk_resource_t *group)
	{
	if (group != NULL) {
	CRM_CHECK((group->variant == pcmk_rsc_variant_group)
	&& (group->variant_opaque != NULL), return NULL);
	return ((group_variant_data_t *) group->variant_opaque)->last_child;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a group flag is set
	*
	* \param[in] group Group resource to check
	* \param[in] flags Flag or flags to check
	*
	* \return true if all \p flags are set for \p group, otherwise false
	*/
	bool
	pe__group_flag_is_set(const pcmk_resource_t *group, uint32_t flags)
	{
	group_variant_data_t *group_data = NULL;

	CRM_CHECK((group != NULL) && (group->variant == pcmk_rsc_variant_group)
	&& (group->variant_opaque != NULL), return false);
	group_data = (group_variant_data_t *) group->variant_opaque;
	return pcmk_all_flags_set(group_data->flags, flags);
	}

	/*!
	* \internal
	* \brief Set a (deprecated) group flag
	*
	* \param[in,out] group Group resource to check
	* \param[in] option Name of boolean configuration option
	* \param[in] flag Flag to set if \p option is true (which is default)
	* \param[in] wo_bit "Warn once" flag to use for deprecation warning
	*/
	static void
	set_group_flag(pcmk_resource_t group, const char option, uint32_t flag,
	uint32_t wo_bit)
	{
	const char *value_s = NULL;
	int value = 0;

	value_s = g_hash_table_lookup(group->meta, option);

	// We don't actually need the null check but it speeds up the common case
	if ((value_s == NULL) \|\| (crm_str_to_boolean(value_s, &value) < 0)
	\|\| (value != 0)) {

	((group_variant_data_t *) group->variant_opaque)->flags \|= flag;

	} else {
	pcmk__warn_once(wo_bit,
	"Support for the '%s' group meta-attribute is "
	"deprecated and will be removed in a future release "
	"(use a resource set instead)", option);
	}
	}

	static int
	inactive_resources(pcmk_resource_t *rsc)
	{
	int retval = 0;

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (!child_rsc->fns->active(child_rsc, TRUE)) {
	retval++;
	}
	}

	return retval;
	}

	static void
	group_header(pcmk__output_t out, int rc, const pcmk_resource_t *rsc,
	int n_inactive, bool show_inactive, const char *desc)
	{
	GString *attrs = NULL;

	if (n_inactive > 0 && !show_inactive) {
	attrs = g_string_sized_new(64);
	g_string_append_printf(attrs, "%d member%s inactive", n_inactive,
	pcmk__plural_s(n_inactive));
	}

	if (pe__resource_is_disabled(rsc)) {
	pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
	}

	if (attrs != NULL) {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Resource Group: %s (%s)%s%s%s",
	rsc->id,
	(const char *) attrs->str, desc ? " (" : "",
	desc ? desc : "", desc ? ")" : "");
	g_string_free(attrs, TRUE);
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Resource Group: %s%s%s%s",
	rsc->id,
	desc ? " (" : "", desc ? desc : "",
	desc ? ")" : "");
	}
	}

	static bool
	skip_child_rsc(pcmk_resource_t rsc, pcmk_resource_t child,
	gboolean parent_passes, GList *only_rsc, uint32_t show_opts)
	{
	bool star_list = pcmk__list_of_1(only_rsc) &&
	pcmk__str_eq("*", g_list_first(only_rsc)->data, pcmk__str_none);
	bool child_filtered = child->fns->is_filtered(child, only_rsc, FALSE);
	bool child_active = child->fns->active(child, FALSE);
	bool show_inactive = pcmk_is_set(show_opts, pcmk_show_inactive_rscs);

	/* If the resource is in only_rsc by name (so, ignoring "*") then allow
	* it regardless of if it's active or not.
	*/
	if (!star_list && !child_filtered) {
	return false;

	} else if (!child_filtered && (child_active \|\| show_inactive)) {
	return false;

	} else if (parent_passes && (child_active \|\| show_inactive)) {
	return false;

	}

	return true;
	}

	gboolean
	group_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = rsc->xml;
	xmlNode *xml_native_rsc = NULL;
	group_variant_data_t *group_data = NULL;
	const char *clone_id = NULL;

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	group_data = pcmk__assert_alloc(1, sizeof(group_variant_data_t));
	group_data->last_child = NULL;
	rsc->variant_opaque = group_data;

	// @COMPAT These are deprecated since 2.1.5
	set_group_flag(rsc, PCMK_META_ORDERED, pcmk__group_ordered,
	pcmk__wo_group_order);
	set_group_flag(rsc, "collocated", pcmk__group_colocated,
	pcmk__wo_group_coloc);

	clone_id = crm_element_value(rsc->xml, PCMK__META_CLONE);

	for (xml_native_rsc = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
	xml_native_rsc != NULL;
	xml_native_rsc = pcmk__xe_next(xml_native_rsc)) {

	if (pcmk__xe_is(xml_native_rsc, PCMK_XE_PRIMITIVE)) {
	pcmk_resource_t *new_rsc = NULL;

	crm_xml_add(xml_native_rsc, PCMK__META_CLONE, clone_id);
	if (pe__unpack_resource(xml_native_rsc, &new_rsc, rsc,
	scheduler) != pcmk_rc_ok) {
	continue;
	}

	rsc->children = g_list_append(rsc->children, new_rsc);
	group_data->last_child = new_rsc;
	pcmk__rsc_trace(rsc, "Added %s member %s", rsc->id, new_rsc->id);
	}
	}

	if (rsc->children == NULL) {
	/* The schema does not allow empty groups, but if validation is
	* disabled, we allow them (members can be added later).
	*
	* @COMPAT At a major release bump, we should consider this a failure so
	* that group methods can assume children is not NULL, and there
	* are no strange effects from phantom groups due to their
	* presence or meta-attributes.
	*/
	pcmk__config_warn("Group %s will be ignored because it does not have "
	"any members", rsc->id);
	}
	return TRUE;
	}

	gboolean
	group_active(pcmk_resource_t *rsc, gboolean all)
	{
	gboolean c_all = TRUE;
	gboolean c_any = FALSE;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (child_rsc->fns->active(child_rsc, all)) {
	c_any = TRUE;
	} else {
	c_all = FALSE;
	}
	}

	if (c_any == FALSE) {
	return FALSE;
	} else if (all && c_all == FALSE) {
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	group_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	GList *gIter = rsc->children;
	char *child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%s<group " PCMK_XA_ID "=\"%s\" ", pre_text, rsc->id);
	status_print("number_resources=\"%d\" ", g_list_length(rsc->children));
	status_print(">\n");

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	}

	status_print("%s</group>\n", pre_text);
	free(child_text);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	group_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	char *child_text = NULL;
	GList *gIter = rsc->children;

	if (pre_text == NULL) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	group_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	child_text = crm_strdup_printf("%s ", pre_text);

	status_print("%sResource Group: %s", pre_text ? pre_text : "", rsc->id);

	if (options & pe_print_html) {
	status_print("\n<ul>\n");

	} else if ((options & pe_print_log) == 0) {
	status_print("\n");
	}

	if (options & pe_print_brief) {
	print_rscs_brief(rsc->children, child_text, options, print_data, TRUE);

	} else {
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (options & pe_print_html) {
	status_print("<li>\n");
	}
	child_rsc->fns->print(child_rsc, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("group", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__group_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	GList *gIter = rsc->children;

	int rc = pcmk_rc_no_output;

	gboolean parent_passes = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (skip_child_rsc(rsc, child_rsc, parent_passes, only_rsc, show_opts)) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	char *count = pcmk__itoa(g_list_length(gIter));
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk_rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
	const char *disabled = pcmk__btoa(pe__resource_is_disabled(rsc));

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_GROUP, 5,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_NUMBER_RESOURCES, count,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_DISABLED, disabled,
	PCMK_XA_DESCRIPTION, desc);
	free(count);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	- out->message(out, crm_map_element_name(child_rsc->xml), show_opts, child_rsc,
	- only_node, only_rsc);
	+ out->message(out, (const char *) child_rsc->xml->name, show_opts,
	+ child_rsc, only_node, only_rsc);
	}

	if (rc == pcmk_rc_ok) {
	pcmk__output_xml_pop_parent(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("group", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__group_default(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	int rc = pcmk_rc_no_output;

	gboolean parent_passes = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	(strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));

	gboolean active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(rsc->children, only_rsc);

	if (rscs != NULL) {
	group_header(out, &rc, rsc, !active && partially_active ? inactive_resources(rsc) : 0,
	pcmk_is_set(show_opts, pcmk_show_inactive_rscs), desc);
	pe__rscs_brief_output(out, rscs, show_opts \| pcmk_show_inactive_rscs);

	rc = pcmk_rc_ok;
	g_list_free(rscs);
	}

	} else {
	for (GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	if (skip_child_rsc(rsc, child_rsc, parent_passes, only_rsc, show_opts)) {
	continue;
	}

	group_header(out, &rc, rsc, !active && partially_active ? inactive_resources(rsc) : 0,
	pcmk_is_set(show_opts, pcmk_show_inactive_rscs), desc);
	- out->message(out, crm_map_element_name(child_rsc->xml), show_opts,
	+ out->message(out, (const char *) child_rsc->xml->name, show_opts,
	child_rsc, only_node, only_rsc);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	return rc;
	}

	void
	group_free(pcmk_resource_t * rsc)
	{
	CRM_CHECK(rsc != NULL, return);

	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	pcmk__rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
	child_rsc->fns->free(child_rsc);
	}

	pcmk__rsc_trace(rsc, "Freeing child list");
	g_list_free(rsc->children);

	common_free(rsc);
	}

	enum rsc_role_e
	group_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e group_role = pcmk_role_unknown;
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) gIter->data;
	enum rsc_role_e role = child_rsc->fns->state(child_rsc, current);

	if (role > group_role) {
	group_role = role;
	}
	}

	pcmk__rsc_trace(rsc, "%s role: %s", rsc->id, pcmk_role_text(group_role));
	return group_role;
	}

	gboolean
	pe__group_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;

	if (check_parent
	&& pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc,
	false)),
	only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else if (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t *child_rsc = iter->data;

	if (!child_rsc->fns->is_filtered(child_rsc, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get maximum group resource instances per node
	*
	* \param[in] rsc Group resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__group_max_per_node(const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));
	return 1U;
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index cfa47af4ad..174f660b3d 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3441 +1,3441 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;
	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	PCMK__NODE_ATTR_SHUTDOWN, PCMK_NODE_ATTR_TERMINATE, \
	PCMK_NODE_ATTR_STANDBY, "#", NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, PCMK_XA_TYPE);
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(node, rsc->children, scheduler, attrname,
	expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, PCMK_XA_NAME);

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry, NULL, NULL, NULL);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	const char *task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(rsc_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__xe_is(rsc_op, PCMK__XE_LRM_RSC_OP)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;

	node = pcmk__xe_create(node, (const char *) key);
	pcmk__xe_set_content(node, "%s", (const char *) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	#define XPATH_STACK "//" PCMK_XE_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);

	if (stack != NULL) {
	return crm_element_value(stack, PCMK_XA_VALUE);
	}
	return PCMK_VALUE_UNKNOWN;
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair(PCMK_XA_INTERVAL, interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_EXEC_TIME, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_QUEUE_TIME, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d",
	PCMK_XA_FAIL_COUNT, failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK_XA_LAST_FAILURE, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
	rsc_id, rsc->migration_threshold, failcount_s,
	lastfail_s? lastfail_s : "");
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online && node->details->expected_up
	&& !pcmk__is_pacemaker_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->details->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static void
	formatted_xml_buf(const pcmk_resource_t rsc, GString xml_buf, bool raw)
	{
	if (raw && (rsc->orig_xml != NULL)) {
	pcmk__xml_string(rsc->orig_xml, pcmk__xml_fmt_pretty, xml_buf, 0);
	} else {
	pcmk__xml_string(rsc->xml, pcmk__xml_fmt_pretty, xml_buf, 0);
	}
	}

	#define XPATH_DC_VERSION "//" PCMK_XE_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pcmk__is_guest_or_bundle_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;
	const pcmk_node_t *host_node = pcmk__current_node(container);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = pcmk__assert_alloc(name_len, sizeof(char));
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...)
	{
	xmlNodePtr xml_node = NULL;
	va_list args;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = pcmk__xe_create(xml_node, tag_name);

	va_start(args, pairs_count);
	while(pairs_count--) {
	const char param_name = va_arg(args, const char );
	const char param_value = va_arg(args, const char );
	if (param_name && param_value) {
	crm_xml_add(xml_node, param_name, param_value);
	}
	};
	va_end(args);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	#ifdef PCMK__COMPAT_2_0
	return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
	#else
	return "in " PCMK__ROLE_PROMOTED " role ";
	#endif
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	char *weight_s = pcmk__itoa(pe_node->weight);

	pcmk__output_create_xml_node(out, PCMK_XE_BAN,
	PCMK_XA_ID, location->id,
	PCMK_XA_RESOURCE, location->rsc->id,
	PCMK_XA_NODE, pe_node->details->uname,
	PCMK_XA_WEIGHT, weight_s,
	PCMK_XA_PROMOTED_ONLY, promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	PCMK__XA_PROMOTED_ONLY_LEGACY, promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(nodes_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources), ndisabled);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ", %d ", nblocked);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "BLOCKED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " from further action due to failure)");

	} else if (ndisabled && !nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ")");

	} else if (!ndisabled && nblocked) {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "BLOCKED");

	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " from further action due to failure)");

	} else {
	child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = NULL;
	xmlNodePtr resources_node = NULL;
	char *s = NULL;

	nodes_node = pcmk__output_create_xml_node(out, PCMK_XE_NODES_CONFIGURED,
	NULL);
	resources_node = pcmk__output_create_xml_node(out,
	PCMK_XE_RESOURCES_CONFIGURED,
	NULL);

	s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, PCMK_XA_NUMBER, s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, PCMK_XA_NUMBER, s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, PCMK_XA_DISABLED, s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, PCMK_XA_BLOCKED, s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Current DC: ");

	if (dc) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s (version %s) -",
	dc_name, pcmk__s(dc_version_s, "unknown"));

	if (mixed_version) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, " MIXED-VERSION");
	}

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " partition");

	if (crm_is_true(quorum)) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " with");

	} else {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, " WITHOUT");
	}

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " quorum");

	} else {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_WARNING);
	pcmk__xe_set_content(child, "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	const char *with_quorum = pcmk__btoa(crm_is_true(quorum));
	const char *mixed_version_s = pcmk__btoa(mixed_version);

	pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
	PCMK_XA_PRESENT, PCMK_VALUE_TRUE,
	PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
	PCMK_XA_NAME, dc->details->uname,
	PCMK_XA_ID, dc->details->id,
	PCMK_XA_WITH_QUORUM, with_quorum,
	PCMK_XA_MIXED_VERSION, mixed_version_s,
	NULL);
	} else {
	pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
	PCMK_XA_PRESENT, PCMK_VALUE_FALSE,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Resource management: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "DISABLED");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child,
	" (the cluster will not attempt to start, stop,"
	" or recover services)");

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Resource management: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "STOPPED");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child,
	" (the cluster will keep all resources stopped)");

	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get readable string representation of a no-quorum policy
	*
	* \param[in] policy No-quorum policy
	*
	* \return String representation of \p policy
	*/
	static const char *
	no_quorum_policy_text(enum pe_quorum_policy policy)
	{
	switch (policy) {
	case pcmk_no_quorum_freeze:
	return PCMK_VALUE_FREEZE;

	case pcmk_no_quorum_stop:
	return PCMK_VALUE_STOP;

	case pcmk_no_quorum_demote:
	return PCMK_VALUE_DEMOTE;

	case pcmk_no_quorum_ignore:
	return PCMK_VALUE_IGNORE;

	case pcmk_no_quorum_fence:
	return PCMK_VALUE_FENCE_LEGACY;

	default:
	return PCMK_VALUE_UNKNOWN;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *stonith_enabled = pcmk__flag_text(scheduler->flags,
	pcmk_sched_fencing_enabled);
	const char *symmetric_cluster =
	pcmk__flag_text(scheduler->flags, pcmk_sched_symmetric_cluster);
	const char *no_quorum_policy =
	no_quorum_policy_text(scheduler->no_quorum_policy);
	const char *maintenance_mode = pcmk__flag_text(scheduler->flags,
	pcmk_sched_in_maintenance);
	const char *stop_all_resources = pcmk__flag_text(scheduler->flags,
	pcmk_sched_stop_all);
	char *stonith_timeout_ms_s = pcmk__itoa(scheduler->stonith_timeout);
	char *priority_fencing_delay_ms_s =
	pcmk__itoa(scheduler->priority_fencing_delay * 1000);

	pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_OPTIONS,
	PCMK_XA_STONITH_ENABLED, stonith_enabled,
	PCMK_XA_SYMMETRIC_CLUSTER, symmetric_cluster,
	PCMK_XA_NO_QUORUM_POLICY, no_quorum_policy,
	PCMK_XA_MAINTENANCE_MODE, maintenance_mode,
	PCMK_XA_STOP_ALL_RESOURCES, stop_all_resources,
	PCMK_XA_STONITH_TIMEOUT_MS,
	stonith_timeout_ms_s,
	PCMK_XA_PRIORITY_FENCING_DELAY_MS,
	priority_fencing_delay_ms_s,
	NULL);
	free(stonith_timeout_ms_s);
	free(priority_fencing_delay_ms_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Stack: ");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s", stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " (");

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s",
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));

	child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, PCMK_XE_STACK,
	PCMK_XA_TYPE, stack_s,
	PCMK_XA_PACEMAKERD_STATE, state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	char *time_s = NULL;

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Last updated: ");

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	time_s = pcmk__epoch2str(NULL, 0);
	pcmk__xe_set_content(child, "%s", time_s);
	free(time_s);

	if (our_nodename != NULL) {
	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, " on ");

	child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s", our_nodename);
	}

	child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "Last change: ");

	child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL, NULL);
	time_s = last_changed_string(last_written, user, client, origin);
	pcmk__xe_set_content(child, "%s", time_s);
	free(time_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, PCMK_XE_LAST_UPDATE,
	PCMK_XA_TIME, time_s,
	PCMK_XA_ORIGIN, our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, PCMK_XE_LAST_CHANGE,
	PCMK_XA_TIME, pcmk__s(last_written, ""),
	PCMK_XA_USER, pcmk__s(user, ""),
	PCMK_XA_CLIENT, pcmk__s(client, ""),
	PCMK_XA_ORIGIN, pcmk__s(origin, ""),
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {

	pcmk__str_update(&rsc_id, "unknown resource");
	pcmk__str_update(&task, "unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " PCMK_XA_LAST_RC_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *exit_reason = crm_element_value(xml_op, PCMK_XA_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *op_key_name = PCMK_XA_OP_KEY;
	int rc;
	int status;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *exitstatus = NULL;
	const char *exit_reason = pcmk__s(crm_element_value(xml_op,
	PCMK_XA_EXIT_REASON),
	"none");
	const char *status_s = NULL;

	time_t epoch = 0;
	gchar *exit_reason_esc = NULL;
	char *rc_s = NULL;
	xmlNodePtr node = NULL;

	if (pcmk__xml_needs_escape(exit_reason, pcmk__xml_escape_attr)) {
	exit_reason_esc = pcmk__xml_escape(exit_reason, pcmk__xml_escape_attr);
	exit_reason = exit_reason_esc;
	}
	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	if (crm_element_value(xml_op, PCMK__XA_OPERATION_KEY) == NULL) {
	op_key_name = PCMK_XA_ID;
	}
	exitstatus = services_ocf_exitcode_str(rc);
	rc_s = pcmk__itoa(rc);
	status_s = pcmk_exec_status_str(status);
	node = pcmk__output_create_xml_node(out, PCMK_XE_FAILURE,
	op_key_name, op_key,
	PCMK_XA_NODE, uname,
	PCMK_XA_EXITSTATUS, exitstatus,
	PCMK_XA_EXITREASON, exit_reason,
	PCMK_XA_EXITCODE, rc_s,
	PCMK_XA_CALL, call_id,
	PCMK_XA_STATUS, status_s,
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {

	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	const char *exec = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, rc_change,
	PCMK_XA_QUEUED, queue_time,
	PCMK_XA_EXEC, exec,
	PCMK_XA_INTERVAL, interval_ms_s,
	PCMK_XA_TASK, task,
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	g_free(exit_reason_esc);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xe_first_child(scheduler->failed, NULL, NULL, NULL);
	xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {

	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, PCMK_XA_UNAME),
	only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pcmk__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);
	xmlNode *child = NULL;

	// Cluster membership
	if (node->details->online) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_ONLINE);
	pcmk__xe_set_content(child, " online");

	} else {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_OFFLINE);
	pcmk__xe_set_content(child, " OFFLINE");
	}

	// Standby mode
	if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	pcmk__xe_set_content(child,
	" (in standby due to " PCMK_META_ON_FAIL ","
	" with active resources)");

	} else if (node->details->standby_onfail) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	pcmk__xe_set_content(child,
	" (in standby due to " PCMK_META_ON_FAIL ")");

	} else if (node->details->standby && (node->details->running_rsc != NULL)) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	pcmk__xe_set_content(child,
	" (in standby, with active resources)");

	} else if (node->details->standby) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK_VALUE_STANDBY);
	pcmk__xe_set_content(child, " (in standby)");
	}

	// Maintenance mode
	if (node->details->maintenance) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_MAINT);
	pcmk__xe_set_content(child, " (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_HEALTH_RED);
	pcmk__xe_set_content(child, " (health is RED)");

	} else if (health == 0) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_HEALTH_YELLOW);
	pcmk__xe_set_content(child, " (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, ", feature set %s", feature_set);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNode *item_node = NULL;
	xmlNode *child = NULL;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	- out->message(out, crm_map_element_name(rsc->xml), show_opts,
	+ out->message(out, pcmk__map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "%s:", node_name);
	status_node(node, item_node, show_opts);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	return "standby (" PCMK_META_ON_FAIL ")";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	return "standby (with active resources)";
	} else {
	return "standby";
	}
	} else {
	return "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pcmk__is_guest_or_bundle_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pcmk__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	- out->message(out, crm_map_element_name(rsc->xml), show_opts,
	- rsc, only_node, only_rsc);
	+ out->message(out, pcmk__map_element_name(rsc->xml),
	+ show_opts, rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Convert an integer health value to a string representation
	*
	* \param[in] health Integer health value
	*
	* \retval \c PCMK_VALUE_RED if \p health is less than 0
	* \retval \c PCMK_VALUE_YELLOW if \p health is equal to 0
	* \retval \c PCMK_VALUE_GREEN if \p health is greater than 0
	*/
	static const char *
	health_text(int health)
	{
	if (health < 0) {
	return PCMK_VALUE_RED;
	} else if (health == 0) {
	return PCMK_VALUE_YELLOW;
	} else {
	return PCMK_VALUE_GREEN;
	}
	}

	/*!
	* \internal
	* \brief Convert a node type to a string representation
	*
	* \param[in] type Node type
	*
	* \retval \c PCMK_VALUE_MEMBER if \p node_type is \c pcmk_node_variant_cluster
	* \retval \c PCMK_VALUE_REMOTE if \p node_type is \c pcmk_node_variant_remote
	* \retval \c PCMK__VALUE_PING if \p node_type is \c node_ping
	* \retval \c PCMK_VALUE_UNKNOWN otherwise
	*/
	static const char *
	node_type_str(enum node_type type)
	{
	switch (type) {
	case pcmk_node_variant_cluster:
	return PCMK_VALUE_MEMBER;
	case pcmk_node_variant_remote:
	return PCMK_VALUE_REMOTE;
	case node_ping:
	return PCMK__VALUE_PING;
	default:
	return PCMK_VALUE_UNKNOWN;
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *online = pcmk__btoa(node->details->online);
	const char *standby = pcmk__btoa(node->details->standby);
	const char *standby_onfail = pcmk__btoa(node->details->standby_onfail);
	const char *maintenance = pcmk__btoa(node->details->maintenance);
	const char *pending = pcmk__btoa(node->details->pending);
	const char *unclean = pcmk__btoa(node->details->unclean);
	const char *health = health_text(pe__node_health(node));
	const char *feature_set = get_node_feature_set(node);
	const char *shutdown = pcmk__btoa(node->details->shutdown);
	const char *expected_up = pcmk__btoa(node->details->expected_up);
	const char *is_dc = pcmk__btoa(node->details->is_dc);
	int length = g_list_length(node->details->running_rsc);
	char *resources_running = pcmk__itoa(length);
	const char *node_type = node_type_str(node->details->type);

	pe__name_and_nvpairs_xml(out, true, PCMK_XE_NODE, 15,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	PCMK_XA_ONLINE, online,
	PCMK_XA_STANDBY, standby,
	PCMK_XA_STANDBY_ONFAIL, standby_onfail,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_PENDING, pending,
	PCMK_XA_UNCLEAN, unclean,
	PCMK_XA_HEALTH, health,
	PCMK_XA_FEATURE_SET, feature_set,
	PCMK_XA_SHUTDOWN, shutdown,
	PCMK_XA_EXPECTED_UP, expected_up,
	PCMK_XA_IS_DC, is_dc,
	PCMK_XA_RESOURCES_RUNNING, resources_running,
	PCMK_XA_TYPE, node_type);

	if (pcmk__is_guest_or_bundle_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, PCMK_XA_ID_AS_RESOURCE,
	node->details->remote_rsc->container->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	- out->message(out, crm_map_element_name(rsc->xml), show_opts,
	+ out->message(out, pcmk__map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}
	}

	free(resources_running);

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, PCMK_XE_NODE,
	PCMK_XA_NAME, node->details->uname,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v = 0;
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNode *child = NULL;

	if (value != NULL) {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
	pcmk__xe_set_content(child, "%s: %s", name, value);

	if (v <= 0) {
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child, "(connectivity is lost)");

	} else if (v < expected_score) {
	child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
	PCMK__VALUE_BOLD);
	pcmk__xe_set_content(child,
	"(connectivity is degraded -- expected %d)",
	expected_score);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t ", "xmlNode ")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pcmk__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->meta,
	PCMK_META_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__trim(ctime(&last_change)),
	exec_time);
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pcmk__xe_history_key(xml_op),
	crm_element_value(xml_op, PCMK_XA_UNAME),
	crm_element_value(xml_op, PCMK__XA_CALL_ID),
	crm_element_value(xml_op, PCMK__XA_RC_CODE),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t ", "xmlNode ")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *rc_s = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	const char *status_s = NULL;
	const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS),
	&status, PCMK_EXEC_UNKNOWN);
	status_s = pcmk_exec_status_str(status);

	node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION,
	PCMK_XA_OP, pcmk__xe_history_key(xml_op),
	PCMK_XA_NODE, uname,
	PCMK_XA_CALL, call_id,
	PCMK_XA_RC, rc_s,
	PCMK_XA_STATUS, status_s,
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_provider);

	char *agent_tuple = crm_strdup_printf("%s:%s:%s",
	class,
	(has_provider? provider : ""),
	kind);

	pcmk__xe_set_props(node,
	PCMK_XA_RSC, rsc_printable_id(rsc),
	PCMK_XA_AGENT, agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *last_rc_change = pcmk__trim(ctime(&last_change));
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, last_rc_change,
	PCMK_XA_EXEC_TIME, exec_time,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_ATTRIBUTE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, PCMK_XA_EXPECTED, buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pcmk__node_attr(node, name, NULL, pcmk__rsc_node_current);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pcmk__node_name(node));

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char *uname = node->details->uname;
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, PCMK_XE_CAPACITY,
	PCMK_XA_NODE, uname,
	PCMK_XA_COMMENT, comment,
	NULL);
	g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNode ", "GList ", "GList *", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = pcmk__xe_first_child(node_state, PCMK__XE_LRM, NULL, NULL);
	lrm_rsc = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCES, NULL, NULL);

	/* Print history of each of the node's resources */
	for (rsc_entry = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCE, NULL,
	NULL);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next_same(rsc_entry)) {

	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (parent->variant == pcmk_rsc_variant_group) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = NULL;

	if (op_list == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, PCMK_XA_ID));

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (node->details->standby_onfail && node->details->online)
	\|\| node->details->standby \|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pcmk__is_guest_or_bundle_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pcmk__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pcmk__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pcmk__is_guest_or_bundle_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	/* PCMK_XE_NODES acts as the list's element name for CLI tools that use
	* pcmk__output_enable_list_element. Otherwise PCMK_XE_NODES is the
	* value of the list's PCMK_XA_NAME attribute.
	*/
	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	PCMK_XE_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	for (node_state = pcmk__xe_first_child(cib_status, PCMK__XE_NODE_STATE,
	NULL, NULL);
	node_state != NULL; node_state = pcmk__xe_next_same(node_state)) {

	pcmk_node_t *node = pe_find_node_id(scheduler->nodes,
	pcmk__xe_id(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_NODE_WEIGHT,
	PCMK_XA_FUNCTION, prefix,
	PCMK_XA_NODE, uname,
	PCMK_XA_SCORE, score,
	NULL);

	if (rsc) {
	crm_xml_add(node, PCMK_XA_ID, rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNode ", "const char ", "const char *", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNode ", "const char ", "const char *", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *rc_s = pcmk__itoa(rc);
	const char *rc_text = services_ocf_exitcode_str(rc);
	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION_HISTORY,
	PCMK_XA_CALL, call_id,
	PCMK_XA_TASK, task,
	PCMK_XA_RC, rc_s,
	PCMK_XA_RC_TEXT, rc_text,
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, PCMK_XA_INTERVAL, s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, PCMK_XA_LAST_RC_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_EXEC_TIME, s);
	free(s);
	}
	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_QUEUE_TIME, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id,
	chosen? chosen->details->uname : "none",
	score);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_PROMOTION_SCORE,
	PCMK_XA_ID, child_rsc->id,
	PCMK_XA_SCORE, score,
	NULL);

	if (chosen) {
	crm_xml_add(node, PCMK_XA_NODE, chosen->details->uname);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	GString *xml_buf = g_string_sized_new(1024);
	bool raw = va_arg(args, int);

	formatted_xml_buf(rsc, xml_buf, raw);

	out->output_xml(out, PCMK_XE_XML, xml_buf->str);

	g_string_free(xml_buf, TRUE);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	pcmk__formatted_printf(out, "Resource XML:\n");
	return resource_config(out, args);
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out,
	PCMK_XE_RESOURCE_HISTORY,
	PCMK_XA_ID, rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_ORPHAN, true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->migration_threshold);

	pcmk__xe_set_props(node,
	PCMK_XA_ORPHAN, PCMK_VALUE_FALSE,
	PCMK_META_MIGRATION_THRESHOLD, migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK_XA_FAIL_COUNT, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK_XA_LAST_FAILURE, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& (rsc->variant == pcmk_rsc_variant_primitive)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	- x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
	+ x = out->message(out, pcmk__map_element_name(rsc->xml), show_opts, rsc,
	only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(xml_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pcmk__node_name(node));

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char *uname = node->details->uname;
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = NULL;

	xml_node = pcmk__output_create_xml_node(out, PCMK_XE_UTILIZATION,
	PCMK_XA_RESOURCE, rsc->id,
	PCMK_XA_NODE, uname,
	PCMK_XA_FUNCTION, fn,
	NULL);
	g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *", "bool", "bool")
	static int
	ticket_default(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );
	bool raw = va_arg(args, int);
	bool details = va_arg(args, int);

	GString *detail_str = NULL;

	if (raw) {
	out->list_item(out, ticket->id, "%s", ticket->id);
	return pcmk_rc_ok;
	}

	if (details && g_hash_table_size(ticket->state) > 0) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;
	bool already_added = false;

	detail_str = g_string_sized_new(100);
	pcmk__g_strcat(detail_str, "\t(", NULL);

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void ) &name, (void ) &value)) {
	if (already_added) {
	g_string_append_printf(detail_str, ", %s=", name);
	} else {
	g_string_append_printf(detail_str, "%s=", name);
	already_added = true;
	}

	if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, "expires", NULL)) {
	char *epoch_str = NULL;
	long long time_ll;

	pcmk__scan_ll(value, &time_ll, 0);
	epoch_str = pcmk__epoch2str((const time_t *) &time_ll, 0);
	pcmk__g_strcat(detail_str, epoch_str, NULL);
	free(epoch_str);
	} else {
	pcmk__g_strcat(detail_str, value, NULL);
	}
	}

	pcmk__g_strcat(detail_str, ")", NULL);
	}

	if (ticket->last_granted > -1) {
	/* Prior to the introduction of the details & raw arguments to this
	* function, last-granted would always be added in this block. We need
	* to preserve that behavior. At the same time, we also need to preserve
	* the existing behavior from crm_ticket, which would include last-granted
	* as part of the (...) detail string.
	*
	* Luckily we can check detail_str - if it's NULL, either there were no
	* details, or we are preserving the previous behavior of this function.
	* If it's not NULL, we are either preserving the previous behavior of
	* crm_ticket or we were given details=true as an argument.
	*/
	if (detail_str == NULL) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s\t%s%s last-granted=\"%s\"",
	ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""),
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s\t%s%s %s",
	ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""),
	detail_str->str);
	}
	} else {
	out->list_item(out, NULL, "%s\t%s%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	detail_str != NULL ? detail_str->str : "");
	}

	if (detail_str != NULL) {
	g_string_free(detail_str, TRUE);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *", "bool", "bool")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );
	bool raw G_GNUC_UNUSED = va_arg(args, int);
	bool details G_GNUC_UNUSED = va_arg(args, int);

	const char *status = NULL;
	const char *standby = pcmk__btoa(ticket->standby);

	xmlNodePtr node = NULL;
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	status = ticket->granted? PCMK_VALUE_GRANTED : PCMK_VALUE_REVOKED;

	node = pcmk__output_create_xml_node(out, PCMK_XE_TICKET,
	PCMK_XA_ID, ticket->id,
	PCMK_XA_STATUS, status,
	PCMK_XA_STANDBY, standby,
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, PCMK_XA_LAST_GRANTED, buf);
	free(buf);
	}

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void ) &name, (void ) &value)) {
	/* PCMK_XA_LAST_GRANTED and "expires" are already added by the check
	* for ticket->last_granted above.
	*/
	if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, PCMK_XA_EXPIRES,
	NULL)) {
	continue;
	}

	crm_xml_add(node, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "GHashTable *", "bool", "bool", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	GHashTable tickets = va_arg(args, GHashTable );
	bool print_spacer = va_arg(args, int);
	bool raw = va_arg(args, int);
	bool details = va_arg(args, int);

	GHashTableIter iter;
	gpointer value;

	if (g_hash_table_size(tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, tickets);
	while (g_hash_table_iter_next(&iter, NULL, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket, raw, details);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_default },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
	index 054874136b..1d2fa7b06d 100644
	--- a/tools/crm_resource_print.c
	+++ b/tools/crm_resource_print.c
	@@ -1,837 +1,838 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm_resource.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>

	#define cons_string(x) x?x:"NA"
	static int
	print_constraint(xmlNode xml_obj, void userdata)
	{
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) userdata;
	pcmk__output_t *out = scheduler->priv;
	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);

	if (id == NULL) {
	return pcmk_rc_ok;
	}

	// @COMPAT PCMK__XE_LIFETIME is deprecated
	lifetime = pcmk__xe_first_child(xml_obj, PCMK__XE_LIFETIME, NULL, NULL);
	if (pe_evaluate_rules(lifetime, NULL, scheduler->now, NULL) == FALSE) {
	return pcmk_rc_ok;
	}

	if (!pcmk__xe_is(xml_obj, PCMK_XE_RSC_COLOCATION)) {
	return pcmk_rc_ok;
	}

	out->info(out, "Constraint %s %s %s %s %s %s %s",
	xml_obj->name,
	cons_string(crm_element_value(xml_obj, PCMK_XA_ID)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_RSC)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_WITH_RSC)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_SCORE)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_RSC_ROLE)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_WITH_RSC_ROLE)));

	return pcmk_rc_ok;
	}

	void
	cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler)
	{
	pcmk__xe_foreach_child(pcmk_find_cib_element(scheduler->input,
	PCMK_XE_CONSTRAINTS),
	NULL, print_constraint, scheduler);
	}

	void
	cli_resource_print_cts(pcmk_resource_t rsc, pcmk__output_t out)
	{
	const char *host = NULL;
	bool needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	pcmk_node_t *node = pcmk__current_node(rsc);

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	needs_quorum = FALSE;
	} else {
	// @TODO check requires in resource meta-data and rsc_defaults
	}

	if (node != NULL) {
	host = node->details->uname;
	}

	out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
	rsc->xml->name, rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print_operations(const char rsc_id, const char host_uname,
	bool active, pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_no_output;
	GList *ops = find_operations(rsc_id, host_uname, active, scheduler);

	if (!ops) {
	return rc;
	}

	out->begin_list(out, NULL, NULL, "Resource Operations");
	rc = pcmk_rc_ok;

	for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;
	out->message(out, "node-and-op", scheduler, xml_op);
	}

	out->end_list(out);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	bool expanded)
	{
	pcmk__output_t *out = scheduler->priv;
	uint32_t show_opts = pcmk_show_pending;
	GList *all = NULL;

	all = g_list_prepend(all, (gpointer) "*");

	out->begin_list(out, NULL, NULL, "Resource Config");
	- out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
	+ out->message(out, pcmk__map_element_name(rsc->xml), show_opts, rsc, all,
	+ all);
	out->message(out, "resource-config", rsc, !expanded);
	out->end_list(out);

	g_list_free(all);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Attributes");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_default(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	if (status == PCMK_EXEC_DONE) {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* returned <exit-code> (<exit-description>[: <exit-reason>])
	*/
	out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, (int) rc, services_ocf_exitcode_str((int) rc),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	} else {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* could not be executed (<execution-status>[: <exit-reason>])
	*/
	out->err(out,
	"Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, pcmk_exec_status_str(status),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_xml(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action G_GNUC_UNUSED = va_arg(args, const char );
	const char name G_GNUC_UNUSED = va_arg(args, const char );
	const char class G_GNUC_UNUSED = va_arg(args, const char );
	const char provider G_GNUC_UNUSED = va_arg(args, const char );
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	char *exit_s = pcmk__itoa(rc);
	const char *message = services_ocf_exitcode_str((int) rc);
	char *status_s = pcmk__itoa(status);
	const char *execution_message = pcmk_exec_status_str(status);

	pcmk__output_create_xml_node(out, PCMK_XE_AGENT_STATUS,
	PCMK_XA_CODE, exit_s,
	PCMK_XA_MESSAGE, message,
	PCMK_XA_EXECUTION_CODE, status_s,
	PCMK_XA_EXECUTION_MESSAGE, execution_message,
	PCMK_XA_REASON, exit_reason,
	NULL);

	free(exit_s);
	free(status_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}
	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_default(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	if (rsc_name == NULL) {
	out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
	name, value);
	} else {
	out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
	rsc_name, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_xml(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_OVERRIDE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (rsc_name != NULL) {
	crm_xml_add(node, PCMK_XA_RSC, rsc_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Properties");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, const char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_default(pcmk__output_t *out, va_list args) {
	int verbose = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, PCMK_XE_OVERRIDES);

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	/* hide output for validate-all if not in verbose */
	if ((verbose == 0)
	&& pcmk__str_eq(action, PCMK_ACTION_VALIDATE_ALL, pcmk__str_casei)) {
	return pcmk_rc_ok;
	}

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = pcmk__xml_parse(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, PCMK_XE_COMMAND, stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_xml(pcmk__output_t *out, va_list args) {
	int verbose G_GNUC_UNUSED = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	xmlNodePtr node = NULL;

	node = pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE_AGENT_ACTION,
	PCMK_XA_ACTION, action,
	PCMK_XA_CLASS, class,
	PCMK_XA_TYPE, type,
	NULL);

	if (rsc_name) {
	crm_xml_add(node, PCMK_XA_RSC, rsc_name);
	}

	crm_xml_add(node, PCMK_XA_PROVIDER, provider);

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, PCMK_XE_OVERRIDES);

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = pcmk__xml_parse(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, PCMK_XE_COMMAND, stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_default(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	if (checks->flags == 0) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Checks");

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
	parent->id, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	out->list_item(out, "check",
	"'%s' cannot run on unhealthy nodes due to "
	PCMK_OPT_NODE_HEALTH_STRATEGY "='%s'",
	parent->id,
	pcmk__cluster_option(checks->rsc->cluster->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_xml(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_CHECK,
	PCMK_XA_ID, parent->id,
	NULL);

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_REMAIN_STOPPED, true);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_PROMOTABLE, false);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_UNMANAGED, true);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	crm_xml_add(node, PCMK_XA_LOCKED_TO_HYPHEN, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_UNHEALTHY, true);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const gchar ")
	static int
	resource_search_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const gchar requested_name = va_arg(args, const gchar );

	bool printed = false;
	int rc = pcmk_rc_no_output;

	if (!out->is_quiet(out) && nodes == NULL) {
	out->err(out, "resource %s is NOT running", requested_name);
	return rc;
	}

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;

	if (!printed) {
	out->begin_list(out, NULL, NULL, "Nodes");
	printed = true;
	rc = pcmk_rc_ok;
	}

	if (out->is_quiet(out)) {
	out->list_item(out, "node", "%s", ni->node_name);
	} else {
	const char *role_text = "";

	if (ni->promoted) {
	#ifdef PCMK__COMPAT_2_0
	role_text = " " PCMK__ROLE_PROMOTED_LEGACY;
	#else
	role_text = " " PCMK__ROLE_PROMOTED;
	#endif
	}
	out->list_item(out, "node", "resource %s is running on: %s%s",
	requested_name, ni->node_name, role_text);
	}
	}

	if (printed) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const gchar ")
	static int
	resource_search_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const gchar requested_name = va_arg(args, const gchar );

	pcmk__output_xml_create_parent(out, PCMK_XE_NODES,
	PCMK_XA_RESOURCE, requested_name,
	NULL);

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;
	xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out,
	PCMK_XE_NODE,
	ni->node_name);

	if (ni->promoted) {
	crm_xml_add(sub_node, PCMK_XA_STATE, "promoted");
	}
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_default(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	out->begin_list(out, NULL, NULL, "Resource Reasons");

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	out->list_item(out, "reason", "Resource %s is not running", rsc->id);
	} else {
	out->list_item(out, "reason", "Resource %s is running", rsc->id);
	}

	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	hosts = NULL;
	}

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	} else {
	out->list_item(out, "reason", "Resource %s is not running on host %s",
	rsc->id, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	out->list_item(out, "reason", "Resource %s is %srunning",
	rsc->id, (hosts? "" : "not "));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_xml(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, PCMK_XE_REASON,
	NULL);

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	const char *running = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	running = pcmk__btoa(hosts != NULL);

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, running,
	NULL);

	cli_resource_check(out, rsc, NULL);
	pcmk__output_xml_pop_parent(out);
	g_list_free(hosts);
	hosts = NULL;
	}

	pcmk__output_xml_pop_parent(out);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	crm_xml_add(xml_node, PCMK_XA_RUNNING_ON, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, PCMK_VALUE_TRUE,
	PCMK_XA_HOST, host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, PCMK_VALUE_FALSE,
	PCMK_XA_HOST, host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	pcmk__output_xml_pop_parent(out);
	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	crm_xml_add(xml_node, PCMK_XA_RUNNING, pcmk__btoa(hosts != NULL));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	static void
	add_resource_name(pcmk_resource_t rsc, pcmk__output_t out)
	{
	if (rsc->children == NULL) {
	/* Sometimes PCMK_XE_RESOURCE might act as a PCMK_XA_NAME instead of an
	* XML element name, depending on whether pcmk__output_enable_list_element
	* was called.
	*/
	out->list_item(out, PCMK_XE_RESOURCE, "%s", rsc->id);
	} else {
	g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
	}
	}

	PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
	static int
	resource_names(pcmk__output_t *out, va_list args) {
	GList resources = va_arg(args, GList );

	if (resources == NULL) {
	out->err(out, "NO resources configured\n");
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Names");
	g_list_foreach(resources, (GFunc) add_resource_name, out);
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "agent-status", "default", agent_status_default },
	{ "agent-status", "xml", agent_status_xml },
	{ "attribute-list", "default", attribute_list_default },
	{ "attribute-list", "text", attribute_list_text },
	{ "override", "default", override_default },
	{ "override", "xml", override_xml },
	{ "property-list", "default", property_list_default },
	{ "property-list", "text", property_list_text },
	{ "resource-agent-action", "default", resource_agent_action_default },
	{ "resource-agent-action", "xml", resource_agent_action_xml },
	{ "resource-check-list", "default", resource_check_list_default },
	{ "resource-check-list", "xml", resource_check_list_xml },
	{ "resource-search-list", "default", resource_search_list_default },
	{ "resource-search-list", "xml", resource_search_list_xml },
	{ "resource-reasons-list", "default", resource_reasons_list_default },
	{ "resource-reasons-list", "xml", resource_reasons_list_xml },
	{ "resource-names-list", "default", resource_names },

	{ NULL, NULL, NULL }
	};

	void
	crm_resource_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jan 25, 5:21 AM (9 h, 21 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1321377
Default Alt Text: (361 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions