No OneTemporary
Actions

Size

324 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/include/crm/common/location_internal.h b/include/crm/common/location_internal.h
	index 155e4f2088..401fd94a9a 100644
	--- a/include/crm/common/location_internal.h
	+++ b/include/crm/common/location_internal.h
	@@ -1,36 +1,36 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_LOCATION_INTERNAL__H
	# define PCMK__CRM_COMMON_LOCATION_INTERNAL__H

	#include <glib.h> // GList

	#include <crm/common/nodes.h> // enum pe_discover_e
	#include <crm/common/resources.h> // enum rsc_role_e
	#include <crm/common/scheduler_types.h> // pcmk_resource_t

	#ifdef __cplusplus
	extern "C" {
	#endif

	//! Location constraint object
	typedef struct {
	char *id; // XML ID of location constraint
	pcmk_resource_t *rsc; // Resource with location preference
	enum rsc_role_e role_filter; // Limit to instances with this role
	enum pe_discover_e discover_mode; // How to probe resource on node
	- GList *node_list_rh; // Affected nodes, with preference score
	+ GList *nodes; // Affected nodes, with preference score
	} pcmk__location_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_LOCATION_INTERNAL__H
	diff --git a/lib/pacemaker/pcmk_output.c b/lib/pacemaker/pcmk_output.c
	index 6b31bc5de5..71ce410c88 100644
	--- a/lib/pacemaker/pcmk_output.c
	+++ b/lib/pacemaker/pcmk_output.c
	@@ -1,2405 +1,2405 @@
	/*
	* Copyright 2019-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/msg_xml.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/pengine/internal.h>
	#include <libxml/tree.h>
	#include <pacemaker-internal.h>

	#include <inttypes.h>
	#include <stdint.h>

	static char *
	colocations_header(pcmk_resource_t rsc, pcmk__colocation_t cons,
	bool dependents) {
	char *retval = NULL;

	if (cons->primary_role > pcmk_role_started) {
	retval = crm_strdup_printf("%s (score=%s, %s role=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	(dependents? "needs" : "with"),
	role2text(cons->primary_role), cons->id);
	} else {
	retval = crm_strdup_printf("%s (score=%s, id=%s)",
	rsc->id, pcmk_readable_score(cons->score),
	cons->id);
	}
	return retval;
	}

	static void
	colocations_xml_node(pcmk__output_t out, pcmk_resource_t rsc,
	pcmk__colocation_t *cons) {
	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, XML_CONS_TAG_RSC_DEPEND,
	"id", cons->id,
	"rsc", cons->dependent->id,
	"with-rsc", cons->primary->id,
	"score",
	pcmk_readable_score(cons->score),
	NULL);

	if (cons->node_attribute) {
	xmlSetProp(node, (pcmkXmlStr) "node-attribute",
	(pcmkXmlStr) cons->node_attribute);
	}

	if (cons->dependent_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) "rsc-role",
	(pcmkXmlStr) role2text(cons->dependent_role));
	}

	if (cons->primary_role != pcmk_role_unknown) {
	xmlSetProp(node, (pcmkXmlStr) "with-rsc-role",
	(pcmkXmlStr) role2text(cons->primary_role));
	}
	}

	static int
	do_locations_list_xml(pcmk__output_t out, pcmk_resource_t rsc,
	bool add_header)
	{
	GList *lpc = NULL;
	GList *list = rsc->rsc_location;
	int rc = pcmk_rc_no_output;

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	- for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
	+ for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	if (add_header) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "locations");
	}

	pcmk__output_create_xml_node(out, XML_CONS_TAG_RSC_LOCATION,
	"node", node->details->uname,
	"rsc", rsc->id,
	"id", cons->id,
	"score",
	pcmk_readable_score(node->weight),
	NULL);
	}
	}

	if (add_header) {
	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	int len = 0;
	char *reason = NULL;
	char *details = NULL;
	bool same_host = false;
	bool same_role = false;
	bool need_role = false;

	static int rsc_width = 5;
	static int detail_width = 5;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	len = strlen(rsc->id);
	if (len > rsc_width) {
	rsc_width = len + 2;
	}

	if ((rsc->role > pcmk_role_started)
	\|\| (rsc->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	if (pe__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->role == rsc->next_role) {
	same_role = true;
	}

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role),
	role2text(rsc->next_role),
	pe__node_name(destination));

	} else if (origin == NULL) {
	/* Starting a resource */
	details = crm_strdup_printf("%s", pe__node_name(destination));

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role),
	pe__node_name(origin));

	} else if (destination == NULL) {
	/* Stopping a resource */
	details = crm_strdup_printf("%s", pe__node_name(origin));

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	details = crm_strdup_printf("%s %s", role2text(rsc->role),
	pe__node_name(origin));

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	details = crm_strdup_printf("%s", pe__node_name(origin));

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", pe__node_name(origin),
	pe__node_name(destination),
	role2text(rsc->role));

	} else if (same_role) {
	/* Moving a normal resource */
	details = crm_strdup_printf("%s -> %s", pe__node_name(origin),
	pe__node_name(destination));

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role),
	role2text(rsc->next_role),
	pe__node_name(origin));

	} else {
	/* Moving and promoting/demoting */
	details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role),
	pe__node_name(origin),
	role2text(rsc->next_role),
	pe__node_name(destination));
	}

	len = strlen(details);
	if (len > detail_width) {
	detail_width = len;
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = crm_strdup_printf("due to %s (blocked)", source->reason);

	} else if (source->reason) {
	reason = crm_strdup_printf("due to %s", source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	reason = strdup("blocked");

	}

	out->list_item(out, NULL, "%-8s %-s ( %s )%s%s",
	change, rsc_width, rsc->id, detail_width, details,
	((reason == NULL)? "" : " "), pcmk__s(reason, ""));

	free(details);
	free(reason);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-action-item", "const char ", "pcmk_resource_t ",
	"pcmk_node_t ", "pcmk_node_t ", "pcmk_action_t *",
	"pcmk_action_t *")
	static int
	rsc_action_item_xml(pcmk__output_t *out, va_list args)
	{
	const char change = va_arg(args, const char );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t origin = va_arg(args, pcmk_node_t );
	pcmk_node_t destination = va_arg(args, pcmk_node_t );
	pcmk_action_t action = va_arg(args, pcmk_action_t );
	pcmk_action_t source = va_arg(args, pcmk_action_t );

	char *change_str = NULL;

	bool same_host = false;
	bool same_role = false;
	bool need_role = false;
	xmlNode *xml = NULL;

	CRM_ASSERT(action);
	CRM_ASSERT(destination != NULL \|\| origin != NULL);

	if (source == NULL) {
	source = action;
	}

	if ((rsc->role > pcmk_role_started)
	\|\| (rsc->next_role > pcmk_role_unpromoted)) {
	need_role = true;
	}

	if (pe__same_node(origin, destination)) {
	same_host = true;
	}

	if (rsc->role == rsc->next_role) {
	same_role = true;
	}

	change_str = g_ascii_strdown(change, -1);
	xml = pcmk__output_create_xml_node(out, "rsc_action",
	"action", change_str,
	"resource", rsc->id,
	NULL);
	g_free(change_str);

	if (need_role && (origin == NULL)) {
	/* Starting and promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	"role", role2text(rsc->role),
	"next-role", role2text(rsc->next_role),
	"dest", destination->details->uname,
	NULL);

	} else if (origin == NULL) {
	/* Starting a resource */
	crm_xml_add(xml, "node", destination->details->uname);

	} else if (need_role && (destination == NULL)) {
	/* Stopping a promotable clone instance */
	pcmk__xe_set_props(xml,
	"role", role2text(rsc->role),
	"node", origin->details->uname,
	NULL);

	} else if (destination == NULL) {
	/* Stopping a resource */
	crm_xml_add(xml, "node", origin->details->uname);

	} else if (need_role && same_role && same_host) {
	/* Recovering, restarting or re-promoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	"role", role2text(rsc->role),
	"source", origin->details->uname,
	NULL);

	} else if (same_role && same_host) {
	/* Recovering or Restarting a normal resource */
	crm_xml_add(xml, "source", origin->details->uname);

	} else if (need_role && same_role) {
	/* Moving a promotable clone instance */
	pcmk__xe_set_props(xml,
	"source", origin->details->uname,
	"dest", destination->details->uname,
	"role", role2text(rsc->role),
	NULL);

	} else if (same_role) {
	/* Moving a normal resource */
	pcmk__xe_set_props(xml,
	"source", origin->details->uname,
	"dest", destination->details->uname,
	NULL);

	} else if (same_host) {
	/* Promoting or demoting a promotable clone instance */
	pcmk__xe_set_props(xml,
	"role", role2text(rsc->role),
	"next-role", role2text(rsc->next_role),
	"source", origin->details->uname,
	NULL);

	} else {
	/* Moving and promoting/demoting */
	pcmk__xe_set_props(xml,
	"role", role2text(rsc->role),
	"source", origin->details->uname,
	"next-role", role2text(rsc->next_role),
	"dest", destination->details->uname,
	NULL);
	}

	if ((source->reason != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_props(xml,
	"reason", source->reason,
	"blocked", "true",
	NULL);

	} else if (source->reason != NULL) {
	crm_xml_add(xml, "reason", source->reason);

	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__xe_set_bool_attr(xml, "blocked", true);

	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use rsc->rsc_cons
	* directly rather than rsc->cmds->this_with_colocations().
	*/
	pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc,
	"Resources %s is colocated with", rsc->id);

	if (pcmk_is_set(cons->primary->flags, pcmk_rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->primary->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->primary, cons, false);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->primary);
	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rsc-is-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rsc_is_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use rsc->rsc_cons
	* directly rather than rsc->cmds->this_with_colocations().
	*/
	pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->primary->flags, pcmk_rsc_detect_loop)) {
	colocations_xml_node(out, cons->primary, cons);
	continue;
	}

	colocations_xml_node(out, cons->primary, cons);
	do_locations_list_xml(out, cons->primary, false);

	if (recursive) {
	out->message(out, "rsc-is-colocated-with-list",
	cons->primary, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->rsc_cons_lhs directly rather than
	* rsc->cmds->with_this_colocations().
	*/
	pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;
	char *hdr = NULL;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources colocated with %s",
	rsc->id);

	if (pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)) {
	out->list_item(out, NULL, "%s (id=%s - loop)",
	cons->dependent->id, cons->id);
	continue;
	}

	hdr = colocations_header(cons->dependent, cons, true);
	out->list_item(out, NULL, "%s", hdr);
	free(hdr);

	// Empty list header for indentation of information about this resource
	out->begin_list(out, NULL, NULL, NULL);

	out->message(out, "locations-list", cons->dependent);
	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}

	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("rscs-colocated-with-list", "pcmk_resource_t *", "bool")
	static int
	rscs_colocated_with_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_detect_loop)) {
	return rc;
	}

	/* We're listing constraints explicitly involving rsc, so use
	* rsc->rsc_cons_lhs directly rather than
	* rsc->cmds->with_this_colocations().
	*/
	pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
	for (GList *lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	if (pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)) {
	colocations_xml_node(out, cons->dependent, cons);
	continue;
	}

	colocations_xml_node(out, cons->dependent, cons);
	do_locations_list_xml(out, cons->dependent, false);

	if (recursive) {
	out->message(out, "rscs-colocated-with-list",
	cons->dependent, recursive);
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );

	GList *lpc = NULL;
	GList *list = rsc->rsc_location;
	int rc = pcmk_rc_no_output;

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	pcmk__location_t *cons = lpc->data;

	GList *lpc2 = NULL;

	- for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
	+ for (lpc2 = cons->nodes; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_node_t node = (pcmk_node_t ) lpc2->data;

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Locations");
	out->list_item(out, NULL, "Node %s (score=%s, id=%s, rsc=%s)",
	pe__node_name(node),
	pcmk_readable_score(node->weight), cons->id,
	rsc->id);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("locations-list", "pcmk_resource_t *")
	static int
	locations_list_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	return do_locations_list_xml(out, rsc, true);
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->cluster);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	out->message(out, "locations-list", rsc);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("locations-and-colocations", "pcmk_resource_t *",
	"bool", "bool")
	static int
	locations_and_colocations_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	bool recursive = va_arg(args, int);
	bool force = va_arg(args, int);

	pcmk__unpack_constraints(rsc->cluster);

	// Constraints apply to group/clone, not member/instance
	if (!force) {
	rsc = uber_parent(rsc);
	}

	pcmk__output_xml_create_parent(out, "constraints", NULL);
	do_locations_list_xml(out, rsc, false);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rscs-colocated-with-list", rsc, recursive);

	pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
	out->message(out, "rsc-is-colocated-with-list", rsc, recursive);

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health(pcmk__output_t *out, va_list args)
	{
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	return out->info(out, "Controller on %s in state %s: %s",
	pcmk__s(host_from, "unknown node"),
	pcmk__s(fsa_state, "unknown"),
	pcmk__s(result, "unknown result"));
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	const char host_from G_GNUC_UNUSED = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result G_GNUC_UNUSED = va_arg(args, const char );

	if (fsa_state != NULL) {
	pcmk__formatted_printf(out, "%s\n", fsa_state);
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("health", "const char ", "const char ", "const char *",
	"const char *")
	static int
	health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	const char host_from = va_arg(args, const char );
	const char fsa_state = va_arg(args, const char );
	const char result = va_arg(args, const char );

	pcmk__output_create_xml_node(out, pcmk__s(sys_from, ""),
	"node_name", pcmk__s(host_from, ""),
	"state", pcmk__s(fsa_state, ""),
	"result", pcmk__s(result, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	int rc = pcmk_rc_ok;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	rc = out->info(out, "Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));

	free(last_updated_s);
	return rc;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_html(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;
	char *msg = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk__pcmkd_state_enum2friendly(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	msg = crm_strdup_printf("Status of %s: '%s' (last updated %s)",
	sys_from, state_s,
	pcmk__s(last_updated_s, "at unknown time"));
	pcmk__output_create_html_node(out, "li", NULL, NULL, msg);

	free(msg);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return pacemakerd_health(out, args);
	} else {
	const char sys_from G_GNUC_UNUSED = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated G_GNUC_UNUSED = va_arg(args, time_t);

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}
	pcmk__formatted_printf(out, "%s\n", state_s);
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("pacemakerd-health", "const char *",
	"enum pcmk_pacemakerd_state", "const char *", "time_t")
	static int
	pacemakerd_health_xml(pcmk__output_t *out, va_list args)
	{
	const char sys_from = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char state_s = va_arg(args, const char );
	time_t last_updated = va_arg(args, time_t);

	char *last_updated_s = NULL;

	if (sys_from == NULL) {
	if (state == pcmk_pacemakerd_state_remote) {
	sys_from = "pacemaker-remoted";
	} else {
	sys_from = CRM_SYSTEM_MCP;
	}
	}

	if (state_s == NULL) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}

	if (last_updated != 0) {
	last_updated_s = pcmk__epoch2str(&last_updated,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);
	}

	pcmk__output_create_xml_node(out, "pacemakerd",
	"sys_from", sys_from,
	"state", state_s,
	"last_updated", last_updated_s,
	NULL);
	free(last_updated_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_default(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	out->list_item(out, NULL, "Testing %s ... %.2f secs", xml_file,
	(end - start) / (float) CLOCKS_PER_SEC);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
	static int
	profile_xml(pcmk__output_t *out, va_list args) {
	const char xml_file = va_arg(args, const char );
	clock_t start = va_arg(args, clock_t);
	clock_t end = va_arg(args, clock_t);

	char *duration = pcmk__ftoa((end - start) / (float) CLOCKS_PER_SEC);

	pcmk__output_create_xml_node(out, "timing",
	"file", xml_file,
	"duration", duration,
	NULL);

	free(duration);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	return out->info(out, "Designated Controller is: %s",
	pcmk__s(dc, "not yet elected"));
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return dc(out, args);
	} else {
	const char dc = va_arg(args, const char );

	if (dc != NULL) {
	pcmk__formatted_printf(out, "%s\n", pcmk__s(dc, ""));
	return pcmk_rc_ok;
	}
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("dc", "const char *")
	static int
	dc_xml(pcmk__output_t *out, va_list args)
	{
	const char dc = va_arg(args, const char );

	pcmk__output_create_xml_node(out, "dc",
	"node_name", pcmk__s(dc, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export = va_arg(args, int);

	if (bash_export) {
	return out->info(out, "export %s=%s",
	pcmk__s(name, "<null>"), pcmk__s(id, ""));
	} else {
	return out->info(out, "%s node: %s (%s)", type ? type : "cluster",
	pcmk__s(name, "<null>"), pcmk__s(id, "<null>"));
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_text(pcmk__output_t *out, va_list args)
	{
	if (!out->is_quiet(out)) {
	return crmadmin_node(out, args);
	} else {
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id G_GNUC_UNUSED = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__formatted_printf(out, "%s\n", pcmk__s(name, "<null>"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("crmadmin-node", "const char ", "const char ",
	"const char *", "bool")
	static int
	crmadmin_node_xml(pcmk__output_t *out, va_list args)
	{
	const char type = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char id = va_arg(args, const char );
	bool bash_export G_GNUC_UNUSED = va_arg(args, int);

	pcmk__output_create_xml_node(out, "node",
	"type", type ? type : "cluster",
	"name", pcmk__s(name, ""),
	"id", pcmk__s(id, ""),
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const op_digest_cache_t ")
	static int
	digests_text(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const op_digest_cache_t digests = va_arg(args, const op_digest_cache_t );

	char *action_desc = NULL;
	const char *rsc_desc = "unknown resource";
	const char *node_desc = "unknown node";

	if (interval_ms != 0) {
	action_desc = crm_strdup_printf("%ums-interval %s action", interval_ms,
	((task == NULL)? "unknown" : task));
	} else if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	action_desc = strdup("probe action");
	} else {
	action_desc = crm_strdup_printf("%s action",
	((task == NULL)? "unknown" : task));
	}
	if ((rsc != NULL) && (rsc->id != NULL)) {
	rsc_desc = rsc->id;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_desc = node->details->uname;
	}
	out->begin_list(out, NULL, NULL, "Digests for %s %s on %s",
	rsc_desc, action_desc, node_desc);
	free(action_desc);

	if (digests == NULL) {
	out->list_item(out, NULL, "none");
	out->end_list(out);
	return pcmk_rc_ok;
	}
	if (digests->digest_all_calc != NULL) {
	out->list_item(out, NULL, "%s (all parameters)",
	digests->digest_all_calc);
	}
	if (digests->digest_secure_calc != NULL) {
	out->list_item(out, NULL, "%s (non-private parameters)",
	digests->digest_secure_calc);
	}
	if (digests->digest_restart_calc != NULL) {
	out->list_item(out, NULL, "%s (non-reloadable parameters)",
	digests->digest_restart_calc);
	}
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static void
	add_digest_xml(xmlNode parent, const char type, const char *digest,
	xmlNode *digest_source)
	{
	if (digest != NULL) {
	xmlNodePtr digest_xml = create_xml_node(parent, "digest");

	crm_xml_add(digest_xml, "type", ((type == NULL)? "unspecified" : type));
	crm_xml_add(digest_xml, "hash", digest);
	if (digest_source != NULL) {
	add_node_copy(digest_xml, digest_source);
	}
	}
	}

	PCMK__OUTPUT_ARGS("digests", "const pcmk_resource_t ", "const pcmk_node_t ",
	"const char ", "guint", "const op_digest_cache_t ")
	static int
	digests_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const pcmk_node_t node = va_arg(args, const pcmk_node_t );
	const char task = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);
	const op_digest_cache_t digests = va_arg(args, const op_digest_cache_t );

	char *interval_s = crm_strdup_printf("%ums", interval_ms);
	xmlNode *xml = NULL;

	xml = pcmk__output_create_xml_node(out, "digests",
	"resource", pcmk__s(rsc->id, ""),
	"node",
	pcmk__s(node->details->uname, ""),
	"task", pcmk__s(task, ""),
	"interval", interval_s,
	NULL);
	free(interval_s);
	if (digests != NULL) {
	add_digest_xml(xml, "all", digests->digest_all_calc,
	digests->params_all);
	add_digest_xml(xml, "nonprivate", digests->digest_secure_calc,
	digests->params_secure);
	add_digest_xml(xml, "nonreloadable", digests->digest_restart_calc,
	digests->params_restart);
	}
	return pcmk_rc_ok;
	}

	#define STOP_SANITY_ASSERT(lineno) do { \
	if ((current != NULL) && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if (stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", \
	__func__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if (pcmk_is_set(stop->flags, pcmk_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", \
	__func__, lineno, stop->uuid); \
	CRM_ASSERT(!pcmk_is_set(stop->flags, pcmk_action_optional));\
	} \
	} while (0)

	PCMK__OUTPUT_ARGS("rsc-action", "pcmk_resource_t ", "pcmk_node_t ",
	"pcmk_node_t *")
	static int
	rsc_action_default(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t current = va_arg(args, pcmk_node_t );
	pcmk_node_t next = va_arg(args, pcmk_node_t );

	GList *possible_matches = NULL;
	char *key = NULL;
	int rc = pcmk_rc_no_output;
	bool moving = false;

	pcmk_node_t *start_node = NULL;
	pcmk_action_t *start = NULL;
	pcmk_action_t *stop = NULL;
	pcmk_action_t *promote = NULL;
	pcmk_action_t *demote = NULL;
	pcmk_action_t *reason_op = NULL;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	const bool managed = pcmk_is_set(rsc->flags, pcmk_rsc_managed);

	pe_rsc_info(rsc, "Leave %s\t(%s%s)",
	rsc->id, role2text(rsc->role),
	(managed? "" : " unmanaged"));
	return rc;
	}

	moving = (current != NULL) && (next != NULL)
	&& !pe__same_node(current, next);

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_START,
	false);
	if (possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	if ((start == NULL)
	\|\| !pcmk_is_set(start->flags, pcmk_action_runnable)) {
	start_node = NULL;
	} else {
	start_node = current;
	}
	possible_matches = pe__resource_actions(rsc, start_node, PCMK_ACTION_STOP,
	false);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_stop_unexpected)) {
	/* The resource is multiply active with multiple-active set to
	* stop_unexpected, and not stopping on its current node, but it should
	* be stopping elsewhere.
	*/
	possible_matches = pe__resource_actions(rsc, NULL, PCMK_ACTION_STOP,
	false);
	if (possible_matches != NULL) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_PROMOTE,
	false);
	if (possible_matches) {
	promote = possible_matches->data;
	g_list_free(possible_matches);
	}

	possible_matches = pe__resource_actions(rsc, next, PCMK_ACTION_DEMOTE,
	false);
	if (possible_matches) {
	demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->role == rsc->next_role) {
	pcmk_action_t *migrate_op = NULL;

	CRM_CHECK(next != NULL, return rc);

	possible_matches = pe__resource_actions(rsc, next,
	PCMK_ACTION_MIGRATE_FROM,
	false);
	if (possible_matches) {
	migrate_op = possible_matches->data;
	}

	if ((migrate_op != NULL) && (current != NULL)
	&& pcmk_is_set(migrate_op->flags, pcmk_action_runnable)) {
	rc = out->message(out, "rsc-action-item", "Migrate", rsc, current,
	next, start, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current,
	next, start, NULL);

	} else if ((start == NULL)
	\|\| pcmk_is_set(start->flags, pcmk_action_optional)) {
	if ((demote != NULL) && (promote != NULL)
	&& !pcmk_is_set(demote->flags, pcmk_action_optional)
	&& !pcmk_is_set(promote->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Re-promote", rsc,
	current, next, promote, demote);
	} else {
	pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id,
	role2text(rsc->role), pe__node_name(next));
	}

	} else if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	if ((stop == NULL) \|\| (stop->reason == NULL)) {
	reason_op = start;
	} else {
	reason_op = stop;
	}
	rc = out->message(out, "rsc-action-item", "Stop", rsc, current,
	NULL, stop, reason_op);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving && current) {
	const bool failed = pcmk_is_set(rsc->flags, pcmk_rsc_failed);

	rc = out->message(out, "rsc-action-item",
	(failed? "Recover" : "Move"), rsc, current, next,
	stop, NULL);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	NULL, stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	#if 0
	/* @TODO This can be reached in situations that should really be
	* "Start" (see for example the migrate-fail-7 regression test)
	*/
	STOP_SANITY_ASSERT(__LINE__);
	#endif
	}

	g_list_free(possible_matches);
	return rc;
	}

	if ((stop != NULL)
	&& ((rsc->next_role == pcmk_role_stopped)
	\|\| ((start != NULL)
	&& !pcmk_is_set(start->flags, pcmk_action_runnable)))) {

	key = stop_key(rsc);
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;
	pcmk_action_t *stop_op = NULL;

	reason_op = start;
	possible_matches = find_actions(rsc->actions, key, node);
	if (possible_matches) {
	stop_op = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (stop_op != NULL) {
	if (pcmk_is_set(stop_op->flags, pcmk_action_runnable)) {
	STOP_SANITY_ASSERT(__LINE__);
	}
	if (stop_op->reason != NULL) {
	reason_op = stop_op;
	}
	}

	if (out->message(out, "rsc-action-item", "Stop", rsc, node, NULL,
	stop_op, reason_op) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	free(key);

	} else if ((stop != NULL)
	&& pcmk_all_flags_set(rsc->flags,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed)) {
	/* 'stop' may be NULL if the failure was ignored */
	rc = out->message(out, "rsc-action-item", "Recover", rsc, current,
	next, stop, start);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving) {
	rc = out->message(out, "rsc-action-item", "Move", rsc, current, next,
	stop, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_reload)) {
	rc = out->message(out, "rsc-action-item", "Reload", rsc, current, next,
	start, NULL);

	} else if ((stop != NULL)
	&& !pcmk_is_set(stop->flags, pcmk_action_optional)) {
	rc = out->message(out, "rsc-action-item", "Restart", rsc, current,
	next, start, NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (rsc->role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(current != NULL);
	rc = out->message(out, "rsc-action-item", "Demote", rsc, current,
	next, demote, NULL);

	} else if (rsc->next_role == pcmk_role_promoted) {
	CRM_LOG_ASSERT(next);
	rc = out->message(out, "rsc-action-item", "Promote", rsc, current,
	next, promote, NULL);

	} else if ((rsc->role == pcmk_role_stopped)
	&& (rsc->next_role > pcmk_role_stopped)) {
	rc = out->message(out, "rsc-action-item", "Start", rsc, current, next,
	start, NULL);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	out->list_item(out, NULL, "%s %s '%s'", task, node_name, reason);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-action", "const char ", "const char ", "const char *")
	static int
	node_action_xml(pcmk__output_t *out, va_list args)
	{
	const char task = va_arg(args, const char );
	const char node_name = va_arg(args, const char );
	const char reason = va_arg(args, const char );

	if (task == NULL) {
	return pcmk_rc_no_output;
	} else if (reason) {
	pcmk__output_create_xml_node(out, "node_action",
	"task", task,
	"node", node_name,
	"reason", reason,
	NULL);
	} else {
	crm_notice(" * %s %s", task, node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_default(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	return out->info(out,
	"Node %" PRIu32 ": %s "
	"(uuid=%s, state=%s, have_quorum=%s, is_remote=%s)",
	node_id, pcmk__s(node_name, "unknown"),
	pcmk__s(uuid, "unknown"), pcmk__s(state, "unknown"),
	pcmk__btoa(have_quorum), pcmk__btoa(is_remote));
	}

	PCMK__OUTPUT_ARGS("node-info", "uint32_t", "const char ", "const char ",
	"const char *", "bool", "bool")
	static int
	node_info_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );
	const char uuid = va_arg(args, const char );
	const char state = va_arg(args, const char );
	bool have_quorum = (bool) va_arg(args, int);
	bool is_remote = (bool) va_arg(args, int);

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, "node-info",
	"nodeid", id_s,
	XML_ATTR_UNAME, node_name,
	XML_ATTR_ID, uuid,
	PCMK__XA_CRMD, state,
	XML_ATTR_HAVE_QUORUM, pcmk__btoa(have_quorum),
	XML_NODE_IS_REMOTE, pcmk__btoa(is_remote),
	NULL);
	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNodePtr")
	static int
	inject_cluster_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (rsc != NULL) {
	out->list_item(out, NULL, "Cluster action: %s for %s on %s",
	task, ID(rsc), node);
	} else {
	out->list_item(out, NULL, "Cluster action: %s on %s", task, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-cluster-action", "const char ", "const char ",
	"xmlNodePtr")
	static int
	inject_cluster_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );
	xmlNodePtr rsc = va_arg(args, xmlNodePtr);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "cluster_action",
	"task", task,
	"node", node,
	NULL);

	if (rsc) {
	crm_xml_add(xml_node, "id", ID(rsc));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Fencing %s (%s)", target, op);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-fencing-action", "const char ", "const char ")
	static int
	inject_fencing_action_xml(pcmk__output_t *out, va_list args)
	{
	const char target = va_arg(args, const char );
	const char op = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "fencing_action",
	"target", target,
	"op", op,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNodePtr")
	static int
	inject_attr(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	out->list_item(out, NULL, "Injecting attribute %s=%s into %s '%s'",
	name, value, node_path, ID(cib_node));

	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-attr", "const char ", "const char ", "xmlNodePtr")
	static int
	inject_attr_xml(pcmk__output_t *out, va_list args)
	{
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	xmlNodePtr cib_node = va_arg(args, xmlNodePtr);

	xmlChar *node_path = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node_path = xmlGetNodePath(cib_node);

	pcmk__output_create_xml_node(out, "inject_attr",
	"name", name,
	"value", value,
	"node_path", node_path,
	"cib_node", ID(cib_node),
	NULL);
	free(node_path);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Injecting %s into the configuration", spec);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-spec", "const char *")
	static int
	inject_spec_xml(pcmk__output_t *out, va_list args)
	{
	const char spec = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "inject_spec",
	"spec", spec,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Performing Requested Modifications");

	if (quorum) {
	out->list_item(out, NULL, "Setting quorum: %s", quorum);
	}

	if (watchdog) {
	out->list_item(out, NULL, "Setting watchdog: %s", watchdog);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-config", "const char ", "const char ")
	static int
	inject_modify_config_xml(pcmk__output_t *out, va_list args)
	{
	const char quorum = va_arg(args, const char );
	const char watchdog = va_arg(args, const char );

	xmlNodePtr node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	node = pcmk__output_xml_create_parent(out, "modifications", NULL);

	if (quorum) {
	crm_xml_add(node, "quorum", quorum);
	}

	if (watchdog) {
	crm_xml_add(node, "watchdog", watchdog);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Online", pcmk__str_none)) {
	out->list_item(out, NULL, "Bringing node %s online", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Offline", pcmk__str_none)) {
	out->list_item(out, NULL, "Taking node %s offline", node);
	return pcmk_rc_ok;
	} else if (pcmk__str_eq(action, "Failing", pcmk__str_none)) {
	out->list_item(out, NULL, "Failing node %s", node);
	return pcmk_rc_ok;
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("inject-modify-node", "const char ", "const char ")
	static int
	inject_modify_node_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char node = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "modify_node",
	"action", action,
	"node", node,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (pcmk__str_eq(action, "Standby", pcmk__str_none)) {
	out->list_item(out, NULL, "Making ticket %s standby", ticket);
	} else {
	out->list_item(out, NULL, "%s ticket %s", action, ticket);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-modify-ticket", "const char ", "const char ")
	static int
	inject_modify_ticket_xml(pcmk__output_t *out, va_list args)
	{
	const char action = va_arg(args, const char );
	const char ticket = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	pcmk__output_create_xml_node(out, "modify_ticket",
	"action", action,
	"ticket", ticket,
	NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	out->list_item(out, NULL, "Pseudo action: %s%s%s",
	task, ((node == NULL)? "" : " on "), pcmk__s(node, ""));
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-pseudo-action", "const char ", "const char ")
	static int
	inject_pseudo_action_xml(pcmk__output_t *out, va_list args)
	{
	const char node = va_arg(args, const char );
	const char task = va_arg(args, const char );

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "pseudo_action",
	"task", task,
	NULL);
	if (node) {
	crm_xml_add(xml_node, "node", node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	if (interval_ms) {
	out->list_item(out, NULL, "Resource action: %-15s %s=%u on %s",
	rsc, operation, interval_ms, node);
	} else {
	out->list_item(out, NULL, "Resource action: %-15s %s on %s",
	rsc, operation, node);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("inject-rsc-action", "const char ", "const char ",
	"const char *", "guint")
	static int
	inject_rsc_action_xml(pcmk__output_t *out, va_list args)
	{
	const char rsc = va_arg(args, const char );
	const char operation = va_arg(args, const char );
	const char node = va_arg(args, const char );
	guint interval_ms = va_arg(args, guint);

	xmlNodePtr xml_node = NULL;

	if (out->is_quiet(out)) {
	return pcmk_rc_no_output;
	}

	xml_node = pcmk__output_create_xml_node(out, "rsc_action",
	"resource", rsc,
	"op", operation,
	"node", node,
	NULL);

	if (interval_ms) {
	char *interval_s = pcmk__itoa(interval_ms);

	crm_xml_add(xml_node, "interval", interval_s);
	free(interval_s);
	}

	return pcmk_rc_ok;
	}

	#define CHECK_RC(retcode, retval) \
	if (retval == pcmk_rc_ok) { \
	retcode = pcmk_rc_ok; \
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	int
	pcmk__cluster_status_text(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	int rc = pcmk_rc_no_output;
	bool already_printed_failure = false;

	CHECK_RC(rc, out->message(out, "cluster-summary", scheduler, pcmkd_state,
	section_opts, show_opts));

	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	CHECK_RC(rc, out->message(out, "node-list", scheduler->nodes, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	CHECK_RC(rc, out->message(out, "resource-list", scheduler, show_opts,
	true, unames, resources, rc == pcmk_rc_ok));
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	CHECK_RC(rc, out->message(out, "node-attribute-list", scheduler,
	show_opts, (rc == pcmk_rc_ok), unames,
	resources));
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	CHECK_RC(rc, out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts,
	(rc == pcmk_rc_ok)));
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	CHECK_RC(rc, out->message(out, "failed-action-list", scheduler, unames,
	resources, show_opts, rc == pcmk_rc_ok));
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "failed-fencing-list",
	stonith_history, unames, section_opts,
	show_opts, rc == pcmk_rc_ok));
	}
	} else {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);

	already_printed_failure = true;
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	CHECK_RC(rc, out->message(out, "ticket-list", scheduler,
	(rc == pcmk_rc_ok)));
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	CHECK_RC(rc, out->message(out, "ban-list", scheduler, prefix, resources,
	show_opts, rc == pcmk_rc_ok));
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	CHECK_RC(rc, out->message(out, "fencing-list", hp, unames,
	section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	CHECK_RC(rc, out->message(out, "pending-fencing-list", hp,
	unames, section_opts, show_opts,
	rc == pcmk_rc_ok));
	}
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODES */
	if (pcmk_is_set(section_opts, pcmk_section_nodes)) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	/* XML output always displays full details. */
	uint32_t full_show_opts = show_opts & ~pcmk_show_brief;

	out->message(out, "resource-list", scheduler, full_show_opts,
	false, unames, resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print stonith history */
	if (pcmk_is_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	out->message(out, "full-fencing-list", history_rc, stonith_history,
	unames, section_opts, show_opts, false);
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-status", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "crm_exit_t",
	"stonith_history_t *", "enum pcmk__fence_history", "uint32_t",
	"uint32_t", "const char ", "GList ", "GList *")
	static int
	cluster_status_html(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t stonith_history = va_arg(args, stonith_history_t );
	enum pcmk__fence_history fence_history = va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	const char prefix = va_arg(args, const char );
	GList unames = va_arg(args, GList );
	GList resources = va_arg(args, GList );
	bool already_printed_failure = false;

	out->message(out, "cluster-summary", scheduler, pcmkd_state, section_opts,
	show_opts);

	/* NODE LIST */
	if (pcmk_is_set(section_opts, pcmk_section_nodes) && unames) {
	out->message(out, "node-list", scheduler->nodes, unames, resources,
	show_opts, false);
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(section_opts, pcmk_section_resources)) {
	out->message(out, "resource-list", scheduler, show_opts, true, unames,
	resources, false);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(section_opts, pcmk_section_attributes)) {
	out->message(out, "node-attribute-list", scheduler, show_opts, false,
	unames, resources);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_any_flags_set(section_opts,
	pcmk_section_operations\|pcmk_section_failcounts)) {
	out->message(out, "node-summary", scheduler, unames,
	resources, section_opts, show_opts, false);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(section_opts, pcmk_section_failures)
	&& (scheduler->failed != NULL)
	&& (scheduler->failed->children != NULL)) {

	out->message(out, "failed-action-list", scheduler, unames, resources,
	show_opts, false);
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(section_opts, pcmk_section_fence_failed) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc == 0) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "failed-fencing-list", stonith_history,
	unames, section_opts, show_opts, false);
	}
	} else {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	}

	/* Print stonith history */
	if (pcmk_any_flags_set(section_opts, pcmk_section_fencing_all) &&
	fence_history != pcmk__fence_history_none) {
	if (history_rc != 0) {
	if (!already_printed_failure) {
	out->begin_list(out, NULL, NULL, "Failed Fencing Actions");
	out->list_item(out, NULL, "Failed to get fencing history: %s",
	crm_exit_str(history_rc));
	out->end_list(out);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_worked)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));
	if (hp) {
	out->message(out, "fencing-list", hp, unames, section_opts,
	show_opts, false);
	}
	} else if (pcmk_is_set(section_opts, pcmk_section_fence_pending)) {
	stonith_history_t *hp = NULL;

	hp = stonith__first_matching_event(stonith_history,
	stonith__event_state_pending,
	NULL);
	if (hp) {
	out->message(out, "pending-fencing-list", hp, unames,
	section_opts, show_opts, false);
	}
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(section_opts, pcmk_section_tickets)) {
	out->message(out, "ticket-list", scheduler, false);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(section_opts, pcmk_section_bans)) {
	out->message(out, "ban-list", scheduler, prefix, resources, show_opts,
	false);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ")
	static int
	attribute_default(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );

	GString *s = g_string_sized_new(50);

	if (!pcmk__str_empty(scope)) {
	pcmk__g_strcat(s, "scope=\"", scope, "\" ", NULL);
	}

	if (!pcmk__str_empty(instance)) {
	pcmk__g_strcat(s, "id=\"", instance, "\" ", NULL);
	}

	pcmk__g_strcat(s, "name=\"", pcmk__s(name, ""), "\" ", NULL);

	if (!pcmk__str_empty(host)) {
	pcmk__g_strcat(s, "host=\"", host, "\" ", NULL);
	}

	pcmk__g_strcat(s, "value=\"", pcmk__s(value, ""), "\"", NULL);

	out->info(out, "%s", s->str);
	g_string_free(s, TRUE);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute", "const char ", "const char ", "const char *",
	"const char ", "const char ")
	static int
	attribute_xml(pcmk__output_t *out, va_list args)
	{
	const char scope = va_arg(args, const char );
	const char instance = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	const char host = va_arg(args, const char );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "attribute",
	"name", name,
	"value", value ? value : "",
	NULL);

	if (!pcmk__str_empty(scope)) {
	crm_xml_add(node, "scope", scope);
	}

	if (!pcmk__str_empty(instance)) {
	crm_xml_add(node, "id", instance);
	}

	if (!pcmk__str_empty(host)) {
	crm_xml_add(node, "host", host);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_default(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	switch (result) {
	case pcmk_rc_within_range:
	return out->info(out, "Rule %s is still in effect", rule_id);
	case pcmk_rc_ok:
	return out->info(out, "Rule %s satisfies conditions", rule_id);
	case pcmk_rc_after_range:
	return out->info(out, "Rule %s is expired", rule_id);
	case pcmk_rc_before_range:
	return out->info(out, "Rule %s has not yet taken effect", rule_id);
	case pcmk_rc_op_unsatisfied:
	return out->info(out, "Rule %s does not satisfy conditions",
	rule_id);
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("rule-check", "const char ", "int", "const char ")
	static int
	rule_check_xml(pcmk__output_t *out, va_list args)
	{
	const char rule_id = va_arg(args, const char );
	int result = va_arg(args, int);
	const char error = va_arg(args, const char );

	char *rc_str = pcmk__itoa(pcmk_rc2exitc(result));

	pcmk__output_create_xml_node(out, "rule-check",
	"rule-id", rule_id,
	"rc", rc_str,
	NULL);
	free(rc_str);

	switch (result) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	case pcmk_rc_after_range:
	case pcmk_rc_before_range:
	case pcmk_rc_op_unsatisfied:
	return pcmk_rc_ok;
	default:
	out->err(out,
	"Could not determine whether rule %s is in effect: %s",
	rule_id, ((error != NULL)? error : "unexpected error"));
	return pcmk_rc_ok;
	}
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_none(pcmk__output_t *out, va_list args)
	{
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_text(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	static int code_width = 0;

	if (out->is_quiet(out)) {
	/* If out->is_quiet(), don't print the code. Print name and/or desc in a
	* compact format for text output, or print nothing at all for none-type
	* output.
	*/
	if ((name != NULL) && (desc != NULL)) {
	pcmk__formatted_printf(out, "%s - %s\n", name, desc);

	} else if ((name != NULL) \|\| (desc != NULL)) {
	pcmk__formatted_printf(out, "%s\n", ((name != NULL)? name : desc));
	}
	return pcmk_rc_ok;
	}

	/* Get length of longest (most negative) standard Pacemaker return code
	* This should be longer than all the values of any other type of return
	* code.
	*/
	if (code_width == 0) {
	long long most_negative = pcmk_rc_error - (long long) pcmk__n_rc + 1;
	code_width = (int) snprintf(NULL, 0, "%lld", most_negative);
	}

	if ((name != NULL) && (desc != NULL)) {
	static int name_width = 0;

	if (name_width == 0) {
	// Get length of longest standard Pacemaker return code name
	for (int lpc = 0; lpc < pcmk__n_rc; lpc++) {
	int len = (int) strlen(pcmk_rc_name(pcmk_rc_error - lpc));
	name_width = QB_MAX(name_width, len);
	}
	}
	return out->info(out, "% d: %-s %s", code_width, code, name_width,
	name, desc);
	}

	if ((name != NULL) \|\| (desc != NULL)) {
	return out->info(out, "% *d: %s", code_width, code,
	((name != NULL)? name : desc));
	}

	return out->info(out, "% *d", code_width, code);
	}

	PCMK__OUTPUT_ARGS("result-code", "int", "const char ", "const char ")
	static int
	result_code_xml(pcmk__output_t *out, va_list args)
	{
	int code = va_arg(args, int);
	const char name = va_arg(args, const char );
	const char desc = va_arg(args, const char );

	char *code_str = pcmk__itoa(code);

	pcmk__output_create_xml_node(out, "result-code",
	"code", code_str,
	XML_ATTR_NAME, name,
	XML_ATTR_DESC, desc,
	NULL);
	free(code_str);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "attribute", "default", attribute_default },
	{ "attribute", "xml", attribute_xml },
	{ "cluster-status", "default", pcmk__cluster_status_text },
	{ "cluster-status", "html", cluster_status_html },
	{ "cluster-status", "xml", cluster_status_xml },
	{ "crmadmin-node", "default", crmadmin_node },
	{ "crmadmin-node", "text", crmadmin_node_text },
	{ "crmadmin-node", "xml", crmadmin_node_xml },
	{ "dc", "default", dc },
	{ "dc", "text", dc_text },
	{ "dc", "xml", dc_xml },
	{ "digests", "default", digests_text },
	{ "digests", "xml", digests_xml },
	{ "health", "default", health },
	{ "health", "text", health_text },
	{ "health", "xml", health_xml },
	{ "inject-attr", "default", inject_attr },
	{ "inject-attr", "xml", inject_attr_xml },
	{ "inject-cluster-action", "default", inject_cluster_action },
	{ "inject-cluster-action", "xml", inject_cluster_action_xml },
	{ "inject-fencing-action", "default", inject_fencing_action },
	{ "inject-fencing-action", "xml", inject_fencing_action_xml },
	{ "inject-modify-config", "default", inject_modify_config },
	{ "inject-modify-config", "xml", inject_modify_config_xml },
	{ "inject-modify-node", "default", inject_modify_node },
	{ "inject-modify-node", "xml", inject_modify_node_xml },
	{ "inject-modify-ticket", "default", inject_modify_ticket },
	{ "inject-modify-ticket", "xml", inject_modify_ticket_xml },
	{ "inject-pseudo-action", "default", inject_pseudo_action },
	{ "inject-pseudo-action", "xml", inject_pseudo_action_xml },
	{ "inject-rsc-action", "default", inject_rsc_action },
	{ "inject-rsc-action", "xml", inject_rsc_action_xml },
	{ "inject-spec", "default", inject_spec },
	{ "inject-spec", "xml", inject_spec_xml },
	{ "locations-list", "default", locations_list },
	{ "locations-list", "xml", locations_list_xml },
	{ "node-action", "default", node_action },
	{ "node-action", "xml", node_action_xml },
	{ "node-info", "default", node_info_default },
	{ "node-info", "xml", node_info_xml },
	{ "pacemakerd-health", "default", pacemakerd_health },
	{ "pacemakerd-health", "html", pacemakerd_health_html },
	{ "pacemakerd-health", "text", pacemakerd_health_text },
	{ "pacemakerd-health", "xml", pacemakerd_health_xml },
	{ "profile", "default", profile_default, },
	{ "profile", "xml", profile_xml },
	{ "result-code", "none", result_code_none },
	{ "result-code", "text", result_code_text },
	{ "result-code", "xml", result_code_xml },
	{ "rsc-action", "default", rsc_action_default },
	{ "rsc-action-item", "default", rsc_action_item },
	{ "rsc-action-item", "xml", rsc_action_item_xml },
	{ "rsc-is-colocated-with-list", "default", rsc_is_colocated_with_list },
	{ "rsc-is-colocated-with-list", "xml", rsc_is_colocated_with_list_xml },
	{ "rscs-colocated-with-list", "default", rscs_colocated_with_list },
	{ "rscs-colocated-with-list", "xml", rscs_colocated_with_list_xml },
	{ "rule-check", "default", rule_check_default },
	{ "rule-check", "xml", rule_check_xml },
	{ "locations-and-colocations", "default", locations_and_colocations },
	{ "locations-and-colocations", "xml", locations_and_colocations_xml },

	{ NULL, NULL, NULL }
	};

	void
	pcmk__register_lib_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pacemaker/pcmk_sched_group.c b/lib/pacemaker/pcmk_sched_group.c
	index 8d9decc98e..46be6c417a 100644
	--- a/lib/pacemaker/pcmk_sched_group.c
	+++ b/lib/pacemaker/pcmk_sched_group.c
	@@ -1,950 +1,950 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>

	#include <crm/msg_xml.h>

	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Assign a group resource to a node
	*
	* \param[in,out] rsc Group resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and a child of \p rsc can't be
	* assigned to a node, set the child's next role to
	* stopped and update existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__group_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	pcmk_node_t *first_assigned_node = NULL;
	pcmk_resource_t *first_member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return rsc->allocated_to; // Assignment already done
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pe_rsc_debug(rsc, "Assignment dependency loop detected involving %s",
	rsc->id);
	return NULL;
	}

	if (rsc->children == NULL) {
	// No members to assign
	pe__clear_resource_flags(rsc, pcmk_rsc_unassigned);
	return NULL;
	}

	pe__set_resource_flags(rsc, pcmk_rsc_assigning);
	first_member = (pcmk_resource_t *) rsc->children->data;
	rsc->role = first_member->role;

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;
	pcmk_node_t *node = NULL;

	pe_rsc_trace(rsc, "Assigning group %s member %s",
	rsc->id, member->id);
	node = member->cmds->assign(member, prefer, stop_if_fail);
	if (first_assigned_node == NULL) {
	first_assigned_node = node;
	}
	}

	pe__set_next_role(rsc, first_member->next_role, "first group member");
	pe__clear_resource_flags(rsc, pcmk_rsc_assigning\|pcmk_rsc_unassigned);

	if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
	return NULL;
	}
	return first_assigned_node;
	}

	/*!
	* \internal
	* \brief Create a pseudo-operation for a group as an ordering point
	*
	* \param[in,out] group Group resource to create action for
	* \param[in] action Action name
	*
	* \return Newly created pseudo-operation
	*/
	static pcmk_action_t *
	create_group_pseudo_op(pcmk_resource_t group, const char action)
	{
	pcmk_action_t *op = custom_action(group, pcmk__op_key(group->id, action, 0),
	action, NULL, TRUE, group->cluster);
	pe__set_action_flags(op, pcmk_action_pseudo\|pcmk_action_runnable);
	return op;
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given group resource
	*
	* \param[in,out] rsc Group resource to create actions for
	*/
	void
	pcmk__group_create_actions(pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	pe_rsc_trace(rsc, "Creating actions for group %s", rsc->id);

	// Create actions for individual group members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;

	member->cmds->create_actions(member);
	}

	// Create pseudo-actions for group itself to serve as ordering points
	create_group_pseudo_op(rsc, PCMK_ACTION_START);
	create_group_pseudo_op(rsc, PCMK_ACTION_RUNNING);
	create_group_pseudo_op(rsc, PCMK_ACTION_STOP);
	create_group_pseudo_op(rsc, PCMK_ACTION_STOPPED);
	if (crm_is_true(g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROMOTABLE))) {
	create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTE);
	create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTED);
	create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTE);
	create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTED);
	}
	}

	// User data for member_internal_constraints()
	struct member_data {
	// These could be derived from member but this avoids some function calls
	bool ordered;
	bool colocated;
	bool promotable;

	pcmk_resource_t *last_active;
	pcmk_resource_t *previous_member;
	};

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group member
	*
	* \param[in,out] data Group member to create implicit constraints for
	* \param[in,out] user_data Member data (struct member_data *)
	*/
	static void
	member_internal_constraints(gpointer data, gpointer user_data)
	{
	pcmk_resource_t member = (pcmk_resource_t ) data;
	struct member_data member_data = (struct member_data ) user_data;

	// For ordering demote vs demote or stop vs stop
	uint32_t down_flags = pcmk__ar_then_implies_first_graphed;

	// For ordering demote vs demoted or stop vs stopped
	uint32_t post_down_flags = pcmk__ar_first_implies_then_graphed;

	// Create the individual member's implicit constraints
	member->cmds->internal_constraints(member);

	if (member_data->previous_member == NULL) {
	// This is first member
	if (member_data->ordered) {
	pe__set_order_flags(down_flags, pcmk__ar_ordered);
	post_down_flags = pcmk__ar_first_implies_then;
	}

	} else if (member_data->colocated) {
	uint32_t flags = pcmk__coloc_none;

	if (pcmk_is_set(member->flags, pcmk_rsc_critical)) {
	flags \|= pcmk__coloc_influence;
	}

	// Colocate this member with the previous one
	pcmk__new_colocation("#group-members", NULL, INFINITY, member,
	member_data->previous_member, NULL, NULL, flags);
	}

	if (member_data->promotable) {
	// Demote group -> demote member -> group is demoted
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_DEMOTE,
	member, PCMK_ACTION_DEMOTE, down_flags);
	pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
	member->parent, PCMK_ACTION_DEMOTED,
	post_down_flags);

	// Promote group -> promote member -> group is promoted
	pcmk__order_resource_actions(member, PCMK_ACTION_PROMOTE,
	member->parent, PCMK_ACTION_PROMOTED,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_first_implies_then
	\|pcmk__ar_first_implies_then_graphed);
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	pcmk__ar_then_implies_first_graphed);
	}

	// Stop group -> stop member -> group is stopped
	pcmk__order_stops(member->parent, member, down_flags);
	pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
	member->parent, PCMK_ACTION_STOPPED,
	post_down_flags);

	// Start group -> start member -> group is started
	pcmk__order_starts(member->parent, member,
	pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(member, PCMK_ACTION_START,
	member->parent, PCMK_ACTION_RUNNING,
	pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_first_implies_then
	\|pcmk__ar_first_implies_then_graphed);

	if (!member_data->ordered) {
	pcmk__order_starts(member->parent, member,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks
	\|pcmk__ar_then_implies_first_graphed);
	}

	} else if (member_data->previous_member == NULL) {
	pcmk__order_starts(member->parent, member, pcmk__ar_none);
	if (member_data->promotable) {
	pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
	member, PCMK_ACTION_PROMOTE,
	pcmk__ar_none);
	}

	} else {
	// Order this member relative to the previous one

	pcmk__order_starts(member_data->previous_member, member,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks);
	pcmk__order_stops(member, member_data->previous_member,
	pcmk__ar_ordered\|pcmk__ar_intermediate_stop);

	/* In unusual circumstances (such as adding a new member to the middle
	* of a group with unmanaged later members), this member may be active
	* while the previous (new) member is inactive. In this situation, the
	* usual restart orderings will be irrelevant, so we need to order this
	* member's stop before the previous member's start.
	*/
	if ((member->running_on != NULL)
	&& (member_data->previous_member->running_on == NULL)) {
	pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
	member_data->previous_member,
	PCMK_ACTION_START,
	pcmk__ar_then_implies_first
	\|pcmk__ar_unrunnable_first_blocks);
	}

	if (member_data->promotable) {
	pcmk__order_resource_actions(member_data->previous_member,
	PCMK_ACTION_PROMOTE, member,
	PCMK_ACTION_PROMOTE,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks);
	pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
	member_data->previous_member,
	PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
	}
	}

	// Make sure partially active groups shut down in sequence
	if (member->running_on != NULL) {
	if (member_data->ordered && (member_data->previous_member != NULL)
	&& (member_data->previous_member->running_on == NULL)
	&& (member_data->last_active != NULL)
	&& (member_data->last_active->running_on != NULL)) {
	pcmk__order_stops(member, member_data->last_active,
	pcmk__ar_ordered);
	}
	member_data->last_active = member;
	}

	member_data->previous_member = member;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a group resource
	*
	* \param[in,out] rsc Group resource to create implicit constraints for
	*/
	void
	pcmk__group_internal_constraints(pcmk_resource_t *rsc)
	{
	struct member_data member_data = { false, };
	const pcmk_resource_t *top = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	/* Order group pseudo-actions relative to each other for restarting:
	* stop group -> group is stopped -> start group -> group is started
	*/
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_STOPPED,
	pcmk__ar_unrunnable_first_blocks);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
	rsc, PCMK_ACTION_START,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_START,
	rsc, PCMK_ACTION_RUNNING,
	pcmk__ar_unrunnable_first_blocks);

	top = pe__const_top_resource(rsc, false);

	member_data.ordered = pe__group_flag_is_set(rsc, pcmk__group_ordered);
	member_data.colocated = pe__group_flag_is_set(rsc, pcmk__group_colocated);
	member_data.promotable = pcmk_is_set(top->flags, pcmk_rsc_promotable);
	g_list_foreach(rsc->children, member_internal_constraints, &member_data);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint for a group with some other resource, apply the
	* score to the dependent's allowed node scores (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent group resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_group_with(pcmk_resource_t dependent, const pcmk_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	pcmk_resource_t *member = NULL;

	if (dependent->children == NULL) {
	return;
	}

	pe_rsc_trace(primary, "Processing %s (group %s with %s) for dependent",
	colocation->id, dependent->id, primary->id);

	if (pe__group_flag_is_set(dependent, pcmk__group_colocated)) {
	// Colocate first member (internal colocations will handle the rest)
	member = (pcmk_resource_t *) dependent->children->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation between "
	"non-colocated group and %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate each member individually
	for (GList *iter = dependent->children; iter != NULL; iter = iter->next) {
	member = (pcmk_resource_t *) iter->data;
	member->cmds->apply_coloc_score(member, primary, colocation, true);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint for some other resource with a group, apply the
	* score to the dependent's allowed node scores (if we are still placing
	* resources) or priority (if we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary group resource in colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	colocate_with_group(pcmk_resource_t dependent, const pcmk_resource_t primary,
	const pcmk__colocation_t *colocation)
	{
	const pcmk_resource_t *member = NULL;

	pe_rsc_trace(primary,
	"Processing colocation %s (%s with group %s) for primary",
	colocation->id, dependent->id, primary->id);

	if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
	return;
	}

	if (pe__group_flag_is_set(primary, pcmk__group_colocated)) {

	if (colocation->score >= INFINITY) {
	/* For mandatory colocations, the entire group must be assignable
	* (and in the specified role if any), so apply the colocation based
	* on the last member.
	*/
	member = pe__last_group_member(primary);
	} else if (primary->children != NULL) {
	/* For optional colocations, whether the group is partially or fully
	* up doesn't matter, so apply the colocation based on the first
	* member.
	*/
	member = (pcmk_resource_t *) primary->children->data;
	}
	if (member == NULL) {
	return; // Nothing to colocate with
	}

	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	return;
	}

	if (colocation->score >= INFINITY) {
	pcmk__config_err("%s: Cannot perform mandatory colocation with"
	" non-colocated group %s",
	dependent->id, primary->id);
	return;
	}

	// Colocate dependent with each member individually
	for (const GList *iter = primary->children; iter != NULL;
	iter = iter->next) {
	member = iter->data;
	member->cmds->apply_coloc_score(dependent, member, colocation, false);
	}
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__group_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	colocate_group_with(dependent, primary, colocation);

	} else {
	// Method should only be called for primitive dependents
	CRM_ASSERT(dependent->variant == pcmk_rsc_variant_primitive);

	colocate_with_group(dependent, primary, colocation);
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given group resource action
	*
	* \param[in,out] action Group action to get flags for
	* \param[in] node If not NULL, limit effects to this node
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__group_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	// Default flags for a group action
	uint32_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	CRM_ASSERT(action != NULL);

	// Update flags considering each member's own flags for same action
	for (GList *iter = action->rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;

	// Check whether member has the same action
	enum action_tasks task = get_complex_task(member, action->task);
	const char *task_s = task2text(task);
	pcmk_action_t *member_action = find_first_action(member->actions, NULL,
	task_s, node);

	if (member_action != NULL) {
	uint32_t member_flags = member->cmds->action_flags(member_action,
	node);

	// Group action is mandatory if any member action is
	if (pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(member_flags, pcmk_action_optional)) {
	pe_rsc_trace(action->rsc, "%s is mandatory because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_optional);
	pe__clear_action_flags(action, pcmk_action_optional);
	}

	// Group action is unrunnable if any member action is
	if (!pcmk__str_eq(task_s, action->task, pcmk__str_none)
	&& pcmk_is_set(flags, pcmk_action_runnable)
	&& !pcmk_is_set(member_flags, pcmk_action_runnable)) {

	pe_rsc_trace(action->rsc, "%s is unrunnable because %s is",
	action->uuid, member_action->uuid);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_runnable);
	pe__clear_action_flags(action, pcmk_action_runnable);
	}

	/* Group (pseudo-)actions other than stop or demote are unrunnable
	* unless every member will do it.
	*/
	} else if ((task != pcmk_action_stop) && (task != pcmk_action_demote)) {
	pe_rsc_trace(action->rsc,
	"%s is not runnable because %s will not %s",
	action->uuid, member->id, task_s);
	pe__clear_raw_action_flags(flags, "group action",
	pcmk_action_runnable);
	}
	}

	return flags;
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__group_update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	uint32_t changed = pcmk__updated_none;

	// Group method can be called only on behalf of "then" action
	CRM_ASSERT((first != NULL) && (then != NULL) && (then->rsc != NULL)
	&& (scheduler != NULL));

	// Update the actions for the group itself
	changed \|= pcmk__update_ordered_actions(first, then, node, flags, filter,
	type, scheduler);

	// Update the actions for each group member
	for (GList *iter = then->rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;

	pcmk_action_t *member_action = find_first_action(member->actions, NULL,
	then->task, node);

	if (member_action != NULL) {
	changed \|= member->cmds->update_ordered_actions(first,
	member_action, node,
	flags, filter, type,
	scheduler);
	}
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a group's allowed node scores
	*
	* \param[in,out] rsc Group resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*/
	void
	pcmk__group_apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	GList *node_list_orig = NULL;
	GList *node_list_copy = NULL;
	bool reset_scores = true;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (location != NULL));

	- node_list_orig = location->node_list_rh;
	+ node_list_orig = location->nodes;
	node_list_copy = pcmk__copy_node_list(node_list_orig, true);
	reset_scores = pe__group_flag_is_set(rsc, pcmk__group_colocated);

	// Apply the constraint for the group itself (updates node scores)
	pcmk__apply_location(rsc, location);

	// Apply the constraint for each member
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;

	member->cmds->apply_location(member, location);

	if (reset_scores) {
	/* The first member of colocated groups needs to use the original
	* node scores, but subsequent members should work on a copy, since
	* the first member's scores already incorporate theirs.
	*/
	reset_scores = false;
	- location->node_list_rh = node_list_copy;
	+ location->nodes = node_list_copy;
	}
	}

	- location->node_list_rh = node_list_orig;
	+ location->nodes = node_list_orig;
	g_list_free_full(node_list_copy, free);
	}

	// Group implementation of pcmk_assignment_methods_t:colocated_resources()
	GList *
	pcmk__group_colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const pcmk_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if (pe__group_flag_is_set(rsc, pcmk__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	/* This group has colocated members and/or is cloned -- either way,
	* add every child's colocated resources to the list. The first and last
	* members will include the group's own colocations.
	*/
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	member = (const pcmk_resource_t *) iter->data;
	colocated_rscs = member->cmds->colocated_resources(member, orig_rsc,
	colocated_rscs);
	}

	} else if (rsc->children != NULL) {
	/* This group's members are not colocated, and the group is not cloned,
	* so just add the group's own colocations to the list.
	*/
	colocated_rscs = pcmk__colocated_resources(rsc, orig_rsc,
	colocated_rscs);
	}

	return colocated_rscs;
	}

	// Group implementation of pcmk_assignment_methods_t:with_this_colocations()
	void
	pcmk__with_group_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)

	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (list != NULL));

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* "With this" colocations are needed only for the group itself and for its
	* last member. (Previous members will chain via the group internal
	* colocations.)
	*/
	if ((orig_rsc != rsc) && (orig_rsc != pe__last_group_member(rsc))) {
	return;
	}

	pe_rsc_trace(rsc, "Adding 'with %s' colocations to list for %s",
	rsc->id, orig_rsc->id);

	// Add the group's own colocations
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);

	// If cloned, add any relevant colocations with the clone
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc,
	list);
	}

	if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
	// @COMPAT Non-colocated groups are deprecated
	return;
	}

	// Add explicit colocations with the group's (other) children
	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t *member = iter->data;

	if (member != orig_rsc) {
	member->cmds->with_this_colocations(member, orig_rsc, list);
	}
	}
	}

	// Group implementation of pcmk_assignment_methods_t:this_with_colocations()
	void
	pcmk__group_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	const pcmk_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (list != NULL));

	// Ignore empty groups
	if (rsc->children == NULL) {
	return;
	}

	/* "This with" colocations are normally needed only for the group itself and
	* for its first member.
	*/
	if ((rsc == orig_rsc)
	\|\| (orig_rsc == (const pcmk_resource_t *) rsc->children->data)) {
	pe_rsc_trace(rsc, "Adding '%s with' colocations to list for %s",
	rsc->id, orig_rsc->id);

	// Add the group's own colocations
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);

	// If cloned, add any relevant colocations involving the clone
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc,
	list);
	}

	if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
	// @COMPAT Non-colocated groups are deprecated
	return;
	}

	// Add explicit colocations involving the group's (other) children
	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {
	member = iter->data;
	if (member != orig_rsc) {
	member->cmds->this_with_colocations(member, orig_rsc, list);
	}
	}
	return;
	}

	/* Later group members honor the group's colocations indirectly, due to the
	* internal group colocations that chain everything from the first member.
	* However, if an earlier group member is unmanaged, this chaining will not
	* happen, so the group's mandatory colocations must be explicitly added.
	*/
	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	member = iter->data;
	if (orig_rsc == member) {
	break; // We've seen all earlier members, and none are unmanaged
	}

	if (!pcmk_is_set(member->flags, pcmk_rsc_managed)) {
	crm_trace("Adding mandatory '%s with' colocations to list for "
	"member %s because earlier member %s is unmanaged",
	rsc->id, orig_rsc->id, member->id);
	for (const GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
	cons_iter = cons_iter->next) {
	const pcmk__colocation_t *colocation = NULL;

	colocation = (const pcmk__colocation_t *) cons_iter->data;
	if (colocation->score == INFINITY) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	// @TODO Add mandatory (or all?) clone constraints if cloned
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update nodes with scores of colocated resources' nodes
	*
	* Given a table of nodes and a resource, update the nodes' scores with the
	* scores of the best nodes matching the attribute used for each of the
	* resource's relevant colocations.
	*
	* \param[in,out] source_rsc Group resource whose node scores to add
	* \param[in] target_rsc Resource on whose behalf to update \p *nodes
	* \param[in] log_id Resource ID for logs (if \c NULL, use
	* \p source_rsc ID)
	* \param[in,out] nodes Nodes to update (set initial contents to \c NULL
	* to copy allowed nodes from \p source_rsc)
	* \param[in] colocation Original colocation constraint (used to get
	* configured primary resource's stickiness, and
	* to get colocation node attribute; if \c NULL,
	* <tt>source_rsc</tt>'s own matching node scores will
	* not be added, and \p *nodes must be \c NULL as
	* well)
	* \param[in] factor Incorporate scores multiplied by this factor
	* \param[in] flags Bitmask of enum pcmk__coloc_select values
	*
	* \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
	* the \c pcmk__coloc_select_this_with flag are used together (and only by
	* \c cmp_resources()).
	* \note The caller remains responsible for freeing \p *nodes.
	* \note This is the group implementation of
	* \c pcmk_assignment_methods_t:add_colocated_node_scores().
	*/
	void
	pcmk__group_add_colocated_node_scores(pcmk_resource_t *source_rsc,
	const pcmk_resource_t *target_rsc,
	const char log_id, GHashTable *nodes,
	const pcmk__colocation_t *colocation,
	float factor, uint32_t flags)
	{
	pcmk_resource_t *member = NULL;

	CRM_ASSERT((source_rsc != NULL)
	&& (source_rsc->variant == pcmk_rsc_variant_group)
	&& (nodes != NULL)
	&& ((colocation != NULL)
	\|\| ((target_rsc == NULL) && (*nodes == NULL))));

	if (log_id == NULL) {
	log_id = source_rsc->id;
	}

	// Avoid infinite recursion
	if (pcmk_is_set(source_rsc->flags, pcmk_rsc_updating_nodes)) {
	pe_rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
	log_id, source_rsc->id);
	return;
	}
	pe__set_resource_flags(source_rsc, pcmk_rsc_updating_nodes);

	// Ignore empty groups (only possible with schema validation disabled)
	if (source_rsc->children == NULL) {
	return;
	}

	/* Refer the operation to the first or last member as appropriate.
	*
	* cmp_resources() is the only caller that passes a NULL nodes table,
	* and is also the only caller using pcmk__coloc_select_this_with.
	* For "this with" colocations, the last member will recursively incorporate
	* all the other members' "this with" colocations via the internal group
	* colocations (and via the first member, the group's own colocations).
	*
	* For "with this" colocations, the first member works similarly.
	*/
	if (*nodes == NULL) {
	member = pe__last_group_member(source_rsc);
	} else {
	member = source_rsc->children->data;
	}
	pe_rsc_trace(source_rsc, "%s: Merging scores from group %s using member %s "
	"(at %.6f)", log_id, source_rsc->id, member->id, factor);
	member->cmds->add_colocated_node_scores(member, target_rsc, log_id, nodes,
	colocation, factor, flags);
	pe__clear_resource_flags(source_rsc, pcmk_rsc_updating_nodes);
	}

	// Group implementation of pcmk_assignment_methods_t:add_utilization()
	void
	pcmk__group_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList all_rscs,
	GHashTable *utilization)
	{
	pcmk_resource_t *member = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	pe_rsc_trace(orig_rsc, "%s: Adding group %s as colocated utilization",
	orig_rsc->id, rsc->id);
	if (pe__group_flag_is_set(rsc, pcmk__group_colocated)
	\|\| pe_rsc_is_clone(rsc->parent)) {
	// Every group member will be on same node, so sum all members
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	member = (pcmk_resource_t *) iter->data;

	if (pcmk_is_set(member->flags, pcmk_rsc_unassigned)
	&& (g_list_find(all_rscs, member) == NULL)) {
	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}

	} else if (rsc->children != NULL) {
	// Just add first member's utilization
	member = (pcmk_resource_t *) rsc->children->data;
	if ((member != NULL)
	&& pcmk_is_set(member->flags, pcmk_rsc_unassigned)
	&& (g_list_find(all_rscs, member) == NULL)) {

	member->cmds->add_utilization(member, orig_rsc, all_rscs,
	utilization);
	}
	}
	}

	void
	pcmk__group_shutdown_lock(pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_group));

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t member = (pcmk_resource_t ) iter->data;

	member->cmds->shutdown_lock(member);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c
	index 75490b8fa8..0c2805b2a6 100644
	--- a/lib/pacemaker/pcmk_sched_location.c
	+++ b/lib/pacemaker/pcmk_sched_location.c
	@@ -1,676 +1,674 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static int
	get_node_score(const char rule, const char score, bool raw,
	pcmk_node_t node, pcmk_resource_t rsc)
	{
	int score_f = 0;

	if (score == NULL) {
	pe_err("Rule %s: no score specified. Assuming 0.", rule);

	} else if (raw) {
	score_f = char2score(score);

	} else {
	const char *attr_score = NULL;

	attr_score = pe__node_attribute_calculated(node, score, rsc,
	pcmk__rsc_node_current,
	false);

	if (attr_score == NULL) {
	crm_debug("Rule %s: %s did not have a value for %s",
	rule, pe__node_name(node), score);
	score_f = -INFINITY;

	} else {
	crm_debug("Rule %s: %s had value %s for %s",
	rule, pe__node_name(node), attr_score, score);
	score_f = char2score(attr_score);
	}
	}
	return score_f;
	}

	static pcmk__location_t *
	generate_location_rule(pcmk_resource_t rsc, xmlNode rule_xml,
	const char discovery, crm_time_t next_change,
	pe_re_match_data_t *re_match_data)
	{
	const char *rule_id = NULL;
	const char *score = NULL;
	const char *boolean = NULL;
	const char *role = NULL;

	GList *iter = NULL;
	GList *nodes = NULL;

	bool do_and = true;
	bool accept = true;
	bool raw_score = true;
	bool score_allocated = false;

	pcmk__location_t *location_rule = NULL;

	rule_xml = expand_idref(rule_xml, rsc->cluster->input);
	if (rule_xml == NULL) {
	return NULL;
	}

	rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
	boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
	role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);

	crm_trace("Processing rule: %s", rule_id);

	if ((role != NULL) && (text2role(role) == pcmk_role_unknown)) {
	pe_err("Bad role specified for %s: %s", rule_id, role);
	return NULL;
	}

	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
	if (score == NULL) {
	score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
	if (score != NULL) {
	raw_score = false;
	}
	}
	if (pcmk__str_eq(boolean, "or", pcmk__str_casei)) {
	do_and = false;
	}

	location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL);

	if (location_rule == NULL) {
	return NULL;
	}

	if ((re_match_data != NULL) && (re_match_data->nregs > 0)
	&& (re_match_data->pmatch[0].rm_so != -1) && !raw_score) {

	char *result = pe_expand_re_matches(score, re_match_data);

	if (result != NULL) {
	score = result;
	score_allocated = true;
	}
	}

	if (role != NULL) {
	crm_trace("Setting role filter: %s", role);
	location_rule->role_filter = text2role(role);
	if (location_rule->role_filter == pcmk_role_unpromoted) {
	/* Any promotable clone cannot be promoted without being in the
	* unpromoted role first. Ergo, any constraint for the unpromoted
	* role applies to every role.
	*/
	location_rule->role_filter = pcmk_role_unknown;
	}
	}
	if (do_and) {
	nodes = pcmk__copy_node_list(rsc->cluster->nodes, true);
	for (iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;

	node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
	}
	}

	for (iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	int score_f = 0;
	pcmk_node_t *node = iter->data;
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = pe_rsc_params(rsc, node, rsc->cluster),
	.meta = rsc->meta,
	};

	accept = pe_test_rule(rule_xml, node->details->attrs, pcmk_role_unknown,
	rsc->cluster->now, next_change, &match_data);

	crm_trace("Rule %s %s on %s", ID(rule_xml), accept? "passed" : "failed",
	pe__node_name(node));

	score_f = get_node_score(rule_id, score, raw_score, node, rsc);

	if (accept) {
	pcmk_node_t *local = pe_find_node_id(nodes, node->details->id);

	if ((local == NULL) && do_and) {
	continue;

	} else if (local == NULL) {
	local = pe__copy_node(node);
	nodes = g_list_append(nodes, local);
	}

	if (!do_and) {
	local->weight = pcmk__add_scores(local->weight, score_f);
	}
	crm_trace("%s has score %s after %s", pe__node_name(node),
	pcmk_readable_score(local->weight), rule_id);

	} else if (do_and && !accept) {
	// Remove it
	pcmk_node_t *delete = pe_find_node_id(nodes, node->details->id);

	if (delete != NULL) {
	nodes = g_list_remove(nodes, delete);
	crm_trace("%s did not match", pe__node_name(node));
	}
	free(delete);
	}
	}

	if (score_allocated) {
	free((char *)score);
	}

	- location_rule->node_list_rh = nodes;
	- if (location_rule->node_list_rh == NULL) {
	+ location_rule->nodes = nodes;
	+ if (location_rule->nodes == NULL) {
	crm_trace("No matching nodes for rule %s", rule_id);
	return NULL;
	}

	crm_trace("%s: %d nodes matched",
	- rule_id, g_list_length(location_rule->node_list_rh));
	+ rule_id, g_list_length(location_rule->nodes));
	return location_rule;
	}

	static void
	unpack_rsc_location(xmlNode xml_obj, pcmk_resource_t rsc, const char *role,
	const char score, pe_re_match_data_t re_match_data)
	{
	pcmk__location_t *location = NULL;
	const char *rsc_id = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
	const char *discovery = crm_element_value(xml_obj,
	XML_LOCATION_ATTR_DISCOVERY);

	if (rsc == NULL) {
	pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
	"does not exist", id, rsc_id);
	return;
	}

	if (score == NULL) {
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	}

	if ((node != NULL) && (score != NULL)) {
	int score_i = char2score(score);
	pcmk_node_t *match = pe_find_node(rsc->cluster->nodes, node);

	if (!match) {
	return;
	}
	location = pcmk__new_location(id, rsc, score_i, discovery, match);

	} else {
	bool empty = true;
	crm_time_t *next_change = crm_time_new_undefined();

	/* This loop is logically parallel to pe_evaluate_rules(), except
	* instead of checking whether any rule is active, we set up location
	* constraints for each active rule.
	*/
	for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
	rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
	empty = false;
	crm_trace("Unpacking %s/%s", id, ID(rule_xml));
	generate_location_rule(rsc, rule_xml, discovery, next_change,
	re_match_data);
	}

	if (empty) {
	pcmk__config_err("Ignoring constraint '%s' because it contains "
	"no rules", id);
	}

	/* If there is a point in the future when the evaluation of a rule will
	* change, make sure the scheduler is re-run by that time.
	*/
	if (crm_time_is_defined(next_change)) {
	time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(t, rsc->cluster,
	"location rule evaluation");
	}
	crm_time_free(next_change);
	return;
	}

	if (role == NULL) {
	role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
	}

	if ((location != NULL) && (role != NULL)) {
	if (text2role(role) == pcmk_role_unknown) {
	pe_err("Invalid constraint %s: Bad role %s", id, role);
	return;

	} else {
	enum rsc_role_e r = text2role(role);
	switch (r) {
	case pcmk_role_unknown:
	case pcmk_role_started:
	case pcmk_role_unpromoted:
	/* Applies to all */
	location->role_filter = pcmk_role_unknown;
	break;
	default:
	location->role_filter = r;
	break;
	}
	}
	}
	}

	static void
	unpack_simple_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
	const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);

	if (value) {
	pcmk_resource_t *rsc;

	rsc = pcmk__find_constraint_resource(scheduler->resources, value);
	unpack_rsc_location(xml_obj, rsc, NULL, NULL, NULL);
	}

	value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
	if (value) {
	regex_t *r_patt = calloc(1, sizeof(regex_t));
	bool invert = false;

	if (value[0] == '!') {
	value++;
	invert = true;
	}

	if (regcomp(r_patt, value, REG_EXTENDED) != 0) {
	pcmk__config_err("Ignoring constraint '%s' because "
	XML_LOC_ATTR_SOURCE_PATTERN
	" has invalid value '%s'", id, value);
	free(r_patt);
	return;
	}

	for (GList *iter = scheduler->resources; iter != NULL;
	iter = iter->next) {

	pcmk_resource_t *r = iter->data;
	int nregs = 0;
	regmatch_t *pmatch = NULL;
	int status;

	if (r_patt->re_nsub > 0) {
	nregs = r_patt->re_nsub + 1;
	} else {
	nregs = 1;
	}
	pmatch = calloc(nregs, sizeof(regmatch_t));

	status = regexec(r_patt, r->id, nregs, pmatch, 0);

	if (!invert && (status == 0)) {
	pe_re_match_data_t re_match_data = {
	.string = r->id,
	.nregs = nregs,
	.pmatch = pmatch
	};

	crm_debug("'%s' matched '%s' for %s", r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, &re_match_data);

	} else if (invert && (status != 0)) {
	crm_debug("'%s' is an inverted match of '%s' for %s",
	r->id, value, id);
	unpack_rsc_location(xml_obj, r, NULL, NULL, NULL);

	} else {
	crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
	}

	free(pmatch);
	}

	regfree(r_patt);
	free(r_patt);
	}
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_tags(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	const char *id = NULL;
	const char *rsc_id = NULL;
	const char *state = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;
	xmlNode *rsc_set = NULL;

	*expanded_xml = NULL;

	CRM_CHECK(xml_obj != NULL, return EINVAL);

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	// Check whether there are any resource sets with template or tag references
	*expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
	if (*expanded_xml != NULL) {
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");
	return pcmk_rc_ok;
	}

	rsc_id = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
	if (rsc_id == NULL) {
	return pcmk_rc_ok;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", id, rsc_id);
	return pcmk_rc_unpack_error;

	} else if (rsc != NULL) {
	// No template is referenced
	return pcmk_rc_ok;
	}

	state = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);

	*expanded_xml = copy_xml(xml_obj);

	// Convert any template or tag reference into constraint resource_set
	if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, XML_LOC_ATTR_SOURCE,
	false, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return pcmk_rc_unpack_error;
	}

	if (rsc_set != NULL) {
	if (state != NULL) {
	// Move "rsc-role" into converted resource_set as "role" attribute
	crm_xml_add(rsc_set, "role", state);
	xml_remove_prop(*expanded_xml, XML_RULE_ATTR_ROLE);
	}
	crm_log_xml_trace(*expanded_xml, "Expanded rsc_location");

	} else {
	// No sets
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	}

	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	unpack_location_set(xmlNode location, xmlNode set,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_rsc = NULL;
	pcmk_resource_t *resource = NULL;
	const char *set_id;
	const char *role;
	const char *local_score;

	CRM_CHECK(set != NULL, return EINVAL);

	set_id = ID(set);
	if (set_id == NULL) {
	pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
	XML_ATTR_ID " in constraint '%s'",
	pcmk__s(ID(location), "(missing ID)"));
	return pcmk_rc_unpack_error;
	}

	role = crm_element_value(set, "role");
	local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);

	for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	resource = pcmk__find_constraint_resource(scheduler->resources,
	ID(xml_rsc));
	if (resource == NULL) {
	pcmk__config_err("%s: No resource found for %s",
	set_id, ID(xml_rsc));
	return pcmk_rc_unpack_error;
	}

	unpack_rsc_location(location, resource, role, local_score, NULL);
	}

	return pcmk_rc_ok;
	}

	void
	pcmk__unpack_location(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	xmlNode *set = NULL;
	bool any_sets = false;

	xmlNode *orig_xml = NULL;
	xmlNode *expanded_xml = NULL;

	if (unpack_location_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
	return;
	}

	if (expanded_xml) {
	orig_xml = xml_obj;
	xml_obj = expanded_xml;
	}

	for (set = first_named_child(xml_obj, XML_CONS_TAG_RSC_SET); set != NULL;
	set = crm_next_same_xml(set)) {

	any_sets = true;
	set = expand_idref(set, scheduler->input);
	if ((set == NULL) // Configuration error, message already logged
	\|\| (unpack_location_set(xml_obj, set, scheduler) != pcmk_rc_ok)) {

	if (expanded_xml) {
	free_xml(expanded_xml);
	}
	return;
	}
	}

	if (expanded_xml) {
	free_xml(expanded_xml);
	xml_obj = orig_xml;
	}

	if (!any_sets) {
	unpack_simple_location(xml_obj, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Add a new location constraint to scheduler data
	*
	* \param[in] id XML ID of location constraint
	* \param[in,out] rsc Resource in location constraint
	* \param[in] node_score Constraint score
	* \param[in] discover_mode Resource discovery option for constraint
	* \param[in] node Node in constraint (or NULL if rule-based)
	*
	* \return Newly allocated location constraint
	* \note The result will be added to the cluster (via \p rsc) and should not be
	* freed separately.
	*/
	pcmk__location_t *
	pcmk__new_location(const char id, pcmk_resource_t rsc,
	int node_score, const char discover_mode, pcmk_node_t node)
	{
	pcmk__location_t *new_con = NULL;

	if (id == NULL) {
	pe_err("Invalid constraint: no ID specified");
	return NULL;

	} else if (rsc == NULL) {
	pe_err("Invalid constraint %s: no resource specified", id);
	return NULL;

	} else if (node == NULL) {
	CRM_CHECK(node_score == 0, return NULL);
	}

	new_con = calloc(1, sizeof(pcmk__location_t));
	if (new_con != NULL) {
	new_con->id = strdup(id);
	new_con->rsc = rsc;
	- new_con->node_list_rh = NULL;
	+ new_con->nodes = NULL;
	new_con->role_filter = pcmk_role_unknown;

	if (pcmk__str_eq(discover_mode, "always",
	pcmk__str_null_matches\|pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_always;

	} else if (pcmk__str_eq(discover_mode, "never", pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_never;

	} else if (pcmk__str_eq(discover_mode, "exclusive", pcmk__str_casei)) {
	new_con->discover_mode = pcmk_probe_exclusive;
	rsc->exclusive_discover = TRUE;

	} else {
	pe_err("Invalid " XML_LOCATION_ATTR_DISCOVERY " value %s "
	"in location constraint", discover_mode);
	}

	if (node != NULL) {
	pcmk_node_t *copy = pe__copy_node(node);

	copy->weight = node_score;
	- new_con->node_list_rh = g_list_prepend(NULL, copy);
	+ new_con->nodes = g_list_prepend(NULL, copy);
	}

	rsc->cluster->placement_constraints = g_list_prepend(
	rsc->cluster->placement_constraints, new_con);
	rsc->rsc_location = g_list_prepend(rsc->rsc_location, new_con);
	}

	return new_con;
	}

	/*!
	* \internal
	* \brief Apply all location constraints
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__apply_locations(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->placement_constraints;
	iter != NULL; iter = iter->next) {
	pcmk__location_t *location = iter->data;

	location->rsc->cmds->apply_location(location->rsc, location);
	}
	}

	/*!
	* \internal
	* \brief Apply a location constraint to a resource's allowed node scores
	*
	* \param[in,out] rsc Resource to apply constraint to
	* \param[in,out] location Location constraint to apply
	*
	* \note This does not consider the resource's children, so the resource's
	* apply_location() method should be used instead in most cases.
	*/
	void
	pcmk__apply_location(pcmk_resource_t rsc, pcmk__location_t location)
	{
	bool need_role = false;

	CRM_ASSERT((rsc != NULL) && (location != NULL));

	// If a role was specified, ensure constraint is applicable
	need_role = (location->role_filter > pcmk_role_unknown);
	if (need_role && (location->role_filter != rsc->next_role)) {
	pe_rsc_trace(rsc,
	"Not applying %s to %s because role will be %s not %s",
	location->id, rsc->id, role2text(rsc->next_role),
	role2text(location->role_filter));
	return;
	}

	- if (location->node_list_rh == NULL) {
	+ if (location->nodes == NULL) {
	pe_rsc_trace(rsc, "Not applying %s to %s because no nodes match",
	location->id, rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Applying %s%s%s to %s", location->id,
	(need_role? " for role " : ""),
	(need_role? role2text(location->role_filter) : ""), rsc->id);

	- for (GList *iter = location->node_list_rh;
	- iter != NULL; iter = iter->next) {
	-
	+ for (GList *iter = location->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t *node = iter->data;
	pcmk_node_t *allowed_node = g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	if (allowed_node == NULL) {
	pe_rsc_trace(rsc, "* = %d on %s",
	node->weight, pe__node_name(node));
	allowed_node = pe__copy_node(node);
	g_hash_table_insert(rsc->allowed_nodes,
	(gpointer) allowed_node->details->id,
	allowed_node);
	} else {
	pe_rsc_trace(rsc, "* + %d on %s",
	node->weight, pe__node_name(node));
	allowed_node->weight = pcmk__add_scores(allowed_node->weight,
	node->weight);
	}

	if (allowed_node->rsc_discover_mode < location->discover_mode) {
	if (location->discover_mode == pcmk_probe_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	allowed_node->rsc_discover_mode = location->discover_mode;
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
	index aa07dbc435..3d0fe5ec84 100644
	--- a/lib/pacemaker/pcmk_sched_promotable.c
	+++ b/lib/pacemaker/pcmk_sched_promotable.c
	@@ -1,1301 +1,1301 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add implicit promotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in,out] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_promotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Promote clone" -> promote instance -> "clone promoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	pcmk__order_resource_actions(child, PCMK_ACTION_PROMOTE,
	clone, PCMK_ACTION_PROMOTED,
	pcmk__ar_ordered);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(last, PCMK_ACTION_PROMOTE,
	child, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Add implicit demotion ordering for a promotable instance
	*
	* \param[in,out] clone Clone resource
	* \param[in,out] child Instance of \p clone being ordered
	* \param[in] last Previous instance ordered (NULL if \p child is first)
	*/
	static void
	order_instance_demotion(pcmk_resource_t clone, pcmk_resource_t child,
	pcmk_resource_t *last)
	{
	// "Demote clone" -> demote instance -> "clone demoted"
	pcmk__order_resource_actions(clone, PCMK_ACTION_DEMOTE, child,
	PCMK_ACTION_DEMOTE,
	pcmk__ar_then_implies_first_graphed);
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE,
	clone, PCMK_ACTION_DEMOTED,
	pcmk__ar_first_implies_then_graphed);

	// If clone is ordered, order this instance relative to last
	if ((last != NULL) && pe__clone_is_ordered(clone)) {
	pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE, last,
	PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
	}
	}

	/*!
	* \internal
	* \brief Check whether an instance will be promoted or demoted
	*
	* \param[in] rsc Instance to check
	* \param[out] demoting If \p rsc will be demoted, this will be set to true
	* \param[out] promoting If \p rsc will be promoted, this will be set to true
	*/
	static void
	check_for_role_change(const pcmk_resource_t rsc, bool demoting,
	bool *promoting)
	{
	const GList *iter = NULL;

	// If this is a cloned group, check group members recursively
	if (rsc->children != NULL) {
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	check_for_role_change((const pcmk_resource_t *) iter->data,
	demoting, promoting);
	}
	return;
	}

	for (iter = rsc->actions; iter != NULL; iter = iter->next) {
	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (promoting && demoting) {
	return;

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue;

	} else if (pcmk__str_eq(PCMK_ACTION_DEMOTE, action->task,
	pcmk__str_none)) {
	*demoting = true;

	} else if (pcmk__str_eq(PCMK_ACTION_PROMOTE, action->task,
	pcmk__str_none)) {
	*promoting = true;
	}
	}
	}

	/*!
	* \internal
	* \brief Add promoted-role location constraint scores to an instance's priority
	*
	* Adjust a promotable clone instance's promotion priority by the scores of any
	* location constraints in a list that are both limited to the promoted role and
	* for the node where the instance will be placed.
	*
	* \param[in,out] child Promotable clone instance
	* \param[in] location_constraints List of location constraints to apply
	* \param[in] chosen Node where \p child will be placed
	*/
	static void
	apply_promoted_locations(pcmk_resource_t *child,
	const GList *location_constraints,
	const pcmk_node_t *chosen)
	{
	for (const GList *iter = location_constraints; iter; iter = iter->next) {
	const pcmk__location_t *location = iter->data;
	const pcmk_node_t *constraint_node = NULL;

	if (location->role_filter == pcmk_role_promoted) {
	- constraint_node = pe_find_node_id(location->node_list_rh,
	+ constraint_node = pe_find_node_id(location->nodes,
	chosen->details->id);
	}
	if (constraint_node != NULL) {
	int new_priority = pcmk__add_scores(child->priority,
	constraint_node->weight);

	pe_rsc_trace(child,
	"Applying location %s to %s promotion priority on %s: "
	"%s + %s = %s",
	location->id, child->id,
	pe__node_name(constraint_node),
	pcmk_readable_score(child->priority),
	pcmk_readable_score(constraint_node->weight),
	pcmk_readable_score(new_priority));
	child->priority = new_priority;
	}
	}
	}

	/*!
	* \internal
	* \brief Get the node that an instance will be promoted on
	*
	* \param[in] rsc Promotable clone instance to check
	*
	* \return Node that \p rsc will be promoted on, or NULL if none
	*/
	static pcmk_node_t *
	node_to_be_promoted_on(const pcmk_resource_t *rsc)
	{
	pcmk_node_t *node = NULL;
	pcmk_node_t *local_node = NULL;
	const pcmk_resource_t *parent = NULL;

	// If this is a cloned group, bail if any group member can't be promoted
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	if (node_to_be_promoted_on(child) == NULL) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because member %s can't",
	rsc->id, child->id);
	return NULL;
	}
	}

	node = rsc->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pe_rsc_trace(rsc, "%s can't be promoted because it won't be active",
	rsc->id);
	return NULL;

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	if (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted) {
	crm_notice("Unmanaged instance %s will be left promoted on %s",
	rsc->id, pe__node_name(node));
	} else {
	pe_rsc_trace(rsc, "%s can't be promoted because it is unmanaged",
	rsc->id);
	return NULL;
	}

	} else if (rsc->priority < 0) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because its promotion priority %d "
	"is negative",
	rsc->id, rsc->priority);
	return NULL;

	} else if (!pcmk__node_available(node, false, true)) {
	pe_rsc_trace(rsc, "%s can't be promoted because %s can't run resources",
	rsc->id, pe__node_name(node));
	return NULL;
	}

	parent = pe__const_top_resource(rsc, false);
	local_node = g_hash_table_lookup(parent->allowed_nodes, node->details->id);

	if (local_node == NULL) {
	/* It should not be possible for the scheduler to have assigned the
	* instance to a node where its parent is not allowed, but it's good to
	* have a fail-safe.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_warn("%s can't be promoted because %s is not allowed on %s "
	"(scheduler bug?)",
	rsc->id, parent->id, pe__node_name(node));
	} // else the instance is unmanaged and already promoted
	return NULL;

	} else if ((local_node->count >= pe__clone_promoted_node_max(parent))
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pe_rsc_trace(rsc,
	"%s can't be promoted because %s has "
	"maximum promoted instances already",
	rsc->id, pe__node_name(node));
	return NULL;
	}

	return local_node;
	}

	/*!
	* \internal
	* \brief Compare two promotable clone instances by promotion priority
	*
	* \param[in] a First instance to compare
	* \param[in] b Second instance to compare
	*
	* \return A negative number if \p a has higher promotion priority,
	* a positive number if \p b has higher promotion priority,
	* or 0 if promotion priorities are equal
	*/
	static gint
	cmp_promotable_instance(gconstpointer a, gconstpointer b)
	{
	const pcmk_resource_t rsc1 = (const pcmk_resource_t ) a;
	const pcmk_resource_t rsc2 = (const pcmk_resource_t ) b;

	enum rsc_role_e role1 = pcmk_role_unknown;
	enum rsc_role_e role2 = pcmk_role_unknown;

	CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));

	// Check sort index set by pcmk__set_instance_roles()
	if (rsc1->sort_index > rsc2->sort_index) {
	pe_rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(sort index %d > %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return -1;
	} else if (rsc1->sort_index < rsc2->sort_index) {
	pe_rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(sort index %d < %d)",
	rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
	return 1;
	}

	// If those are the same, prefer instance whose current role is higher
	role1 = rsc1->fns->state(rsc1, TRUE);
	role2 = rsc2->fns->state(rsc2, TRUE);
	if (role1 > role2) {
	pe_rsc_trace(rsc1,
	"%s has higher promotion priority than %s "
	"(higher current role)",
	rsc1->id, rsc2->id);
	return -1;
	} else if (role1 < role2) {
	pe_rsc_trace(rsc1,
	"%s has lower promotion priority than %s "
	"(lower current role)",
	rsc1->id, rsc2->id);
	return 1;
	}

	// Finally, do normal clone instance sorting
	return pcmk__cmp_instance(a, b);
	}

	/*!
	* \internal
	* \brief Add a promotable clone instance's sort index to its node's score
	*
	* Add a promotable clone instance's sort index (which sums its promotion
	* preferences and scores of relevant location constraints for the promoted
	* role) to the node score of the instance's assigned node.
	*
	* \param[in] data Promotable clone instance
	* \param[in,out] user_data Clone parent of \p data
	*/
	static void
	add_sort_index_to_node_score(gpointer data, gpointer user_data)
	{
	const pcmk_resource_t child = (const pcmk_resource_t ) data;
	pcmk_resource_t clone = (pcmk_resource_t ) user_data;

	pcmk_node_t *node = NULL;
	const pcmk_node_t *chosen = NULL;

	if (child->sort_index < 0) {
	pe_rsc_trace(clone, "Not adding sort index of %s: negative", child->id);
	return;
	}

	chosen = child->fns->location(child, NULL, FALSE);
	if (chosen == NULL) {
	pe_rsc_trace(clone, "Not adding sort index of %s: inactive", child->id);
	return;
	}

	node = g_hash_table_lookup(clone->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);

	node->weight = pcmk__add_scores(child->sort_index, node->weight);
	pe_rsc_trace(clone,
	"Added cumulative priority of %s (%s) to score on %s (now %s)",
	child->id, pcmk_readable_score(child->sort_index),
	pe__node_name(node), pcmk_readable_score(node->weight));
	}

	/*!
	* \internal
	* \brief Apply colocation to dependent's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's dependent
	*/
	static void
	apply_coloc_to_dependent(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *primary = colocation->primary;
	uint32_t flags = pcmk__coloc_select_default;
	float factor = colocation->score / (float) INFINITY;

	if (colocation->dependent_role != pcmk_role_promoted) {
	return;
	}
	if (colocation->score < INFINITY) {
	flags = pcmk__coloc_select_active;
	}
	pe_rsc_trace(clone, "Applying colocation %s (promoted %s with %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	primary->cmds->add_colocated_node_scores(primary, clone, clone->id,
	&clone->allowed_nodes, colocation,
	factor, flags);
	}

	/*!
	* \internal
	* \brief Apply colocation to primary's node scores if for promoted role
	*
	* \param[in,out] data Colocation constraint to apply
	* \param[in,out] user_data Promotable clone that is constraint's primary
	*/
	static void
	apply_coloc_to_primary(gpointer data, gpointer user_data)
	{
	pcmk__colocation_t *colocation = data;
	pcmk_resource_t *clone = user_data;
	pcmk_resource_t *dependent = colocation->dependent;
	const float factor = colocation->score / (float) INFINITY;
	const uint32_t flags = pcmk__coloc_select_active
	\|pcmk__coloc_select_nonnegative;

	if ((colocation->primary_role != pcmk_role_promoted)
	\|\| !pcmk__colocation_has_influence(colocation, NULL)) {
	return;
	}

	pe_rsc_trace(clone, "Applying colocation %s (%s with promoted %s) @%s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id,
	pcmk_readable_score(colocation->score));
	dependent->cmds->add_colocated_node_scores(dependent, clone, clone->id,
	&clone->allowed_nodes,
	colocation, factor, flags);
	}

	/*!
	* \internal
	* \brief Set clone instance's sort index to its node's score
	*
	* \param[in,out] data Promotable clone instance
	* \param[in] user_data Parent clone of \p data
	*/
	static void
	set_sort_index_to_node_score(gpointer data, gpointer user_data)
	{
	pcmk_resource_t child = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;

	pcmk_node_t *chosen = child->fns->location(child, NULL, FALSE);

	if (!pcmk_is_set(child->flags, pcmk_rsc_managed)
	&& (child->next_role == pcmk_role_promoted)) {
	child->sort_index = INFINITY;
	pe_rsc_trace(clone,
	"Final sort index for %s is INFINITY (unmanaged promoted)",
	child->id);

	} else if ((chosen == NULL) \|\| (child->sort_index < 0)) {
	pe_rsc_trace(clone,
	"Final sort index for %s is %d (ignoring node score)",
	child->id, child->sort_index);

	} else {
	const pcmk_node_t *node = g_hash_table_lookup(clone->allowed_nodes,
	chosen->details->id);

	CRM_ASSERT(node != NULL);
	child->sort_index = node->weight;
	pe_rsc_trace(clone,
	"Adding scores for %s: final sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	}

	/*!
	* \internal
	* \brief Sort a promotable clone's instances by descending promotion priority
	*
	* \param[in,out] clone Promotable clone to sort
	*/
	static void
	sort_promotable_instances(pcmk_resource_t *clone)
	{
	GList *colocations = NULL;

	if (pe__set_clone_flag(clone, pcmk__clone_promotion_constrained)
	== pcmk_rc_already) {
	return;
	}
	pe__set_resource_flags(clone, pcmk_rsc_updating_nodes);

	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	pe_rsc_trace(clone,
	"Adding scores for %s: initial sort index for %s is %d",
	clone->id, child->id, child->sort_index);
	}
	pe__show_node_scores(true, clone, "Before", clone->allowed_nodes,
	clone->cluster);

	g_list_foreach(clone->children, add_sort_index_to_node_score, clone);

	colocations = pcmk__this_with_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_dependent, clone);
	g_list_free(colocations);

	colocations = pcmk__with_this_colocations(clone);
	g_list_foreach(colocations, apply_coloc_to_primary, clone);
	g_list_free(colocations);

	// Ban resource from all nodes if it needs a ticket but doesn't have it
	pcmk__require_promotion_tickets(clone);

	pe__show_node_scores(true, clone, "After", clone->allowed_nodes,
	clone->cluster);

	// Reset sort indexes to final node scores
	g_list_foreach(clone->children, set_sort_index_to_node_score, clone);

	// Finally, sort instances in descending order of promotion priority
	clone->children = g_list_sort(clone->children, cmp_promotable_instance);
	pe__clear_resource_flags(clone, pcmk_rsc_updating_nodes);
	}

	/*!
	* \internal
	* \brief Find the active instance (if any) of an anonymous clone on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return
	*/
	static pcmk_resource_t *
	find_active_anon_instance(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	pcmk_resource_t *active = NULL;

	// Use ->find_rsc() in case this is a cloned group
	active = clone->fns->find_rsc(child, id, node,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node);
	if (active != NULL) {
	return active;
	}
	}
	return NULL;
	}

	/*
	* \brief Check whether an anonymous clone instance is known on a node
	*
	* \param[in] clone Anonymous clone to check
	* \param[in] id Instance ID (without instance number) to check
	* \param[in] node Node to check
	*
	* \return true if \p id instance of \p clone is known on \p node,
	* otherwise false
	*/
	static bool
	anonymous_known_on(const pcmk_resource_t clone, const char id,
	const pcmk_node_t *node)
	{
	for (GList *iter = clone->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;

	/* Use ->find_rsc() because this might be a cloned group, and knowing
	* that other members of the group are known here implies nothing.
	*/
	child = clone->fns->find_rsc(child, id, NULL,
	pcmk_rsc_match_clone_only);
	CRM_LOG_ASSERT(child != NULL);
	if (child != NULL) {
	if (g_hash_table_lookup(child->known_on, node->details->id)) {
	return true;
	}
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node is allowed to run a resource
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is allowed to run \p rsc, otherwise false
	*/
	static bool
	is_allowed(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id);

	return (allowed != NULL) && (allowed->weight >= 0);
	}

	/*!
	* \brief Check whether a clone instance's promotion score should be considered
	*
	* \param[in] rsc Promotable clone instance to check
	* \param[in] node Node where score would be applied
	*
	* \return true if \p rsc's promotion score should be considered on \p node,
	* otherwise false
	*/
	static bool
	promotion_score_applies(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	char *id = clone_strip(rsc->id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
	pcmk_resource_t *active = NULL;
	const char *reason = "allowed";

	// Some checks apply only to anonymous clone instances
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {

	// If instance is active on the node, its score definitely applies
	active = find_active_anon_instance(parent, id, node);
	if (active == rsc) {
	reason = "active";
	goto check_allowed;
	}

	/* If no instance is active on this node, this instance's score will
	* count if it has been probed on this node.
	*/
	if ((active == NULL) && anonymous_known_on(parent, id, node)) {
	reason = "probed";
	goto check_allowed;
	}
	}

	/* If this clone's status is unknown on all nodes (e.g. cluster startup),
	* take all instances' scores into account, to make sure we use any
	* permanent promotion scores.
	*/
	if ((rsc->running_on == NULL) && (g_hash_table_size(rsc->known_on) == 0)) {
	reason = "none probed";
	goto check_allowed;
	}

	/* Otherwise, we've probed and/or started the resource somewhere, so
	* consider promotion scores on nodes where we know the status.
	*/
	if ((g_hash_table_lookup(rsc->known_on, node->details->id) != NULL)
	\|\| (pe_find_node_id(rsc->running_on, node->details->id) != NULL)) {
	reason = "known";
	} else {
	pe_rsc_trace(rsc,
	"Ignoring %s promotion score (for %s) on %s: not probed",
	rsc->id, id, pe__node_name(node));
	free(id);
	return false;
	}

	check_allowed:
	if (is_allowed(rsc, node)) {
	pe_rsc_trace(rsc, "Counting %s promotion score (for %s) on %s: %s",
	rsc->id, id, pe__node_name(node), reason);
	free(id);
	return true;
	}

	pe_rsc_trace(rsc, "Ignoring %s promotion score (for %s) on %s: not allowed",
	rsc->id, id, pe__node_name(node));
	free(id);
	return false;
	}

	/*!
	* \internal
	* \brief Get the value of a promotion score node attribute
	*
	* \param[in] rsc Promotable clone instance to get promotion score for
	* \param[in] node Node to get promotion score for
	* \param[in] name Resource name to use in promotion score attribute name
	*
	* \return Value of promotion score node attribute for \p rsc on \p node
	*/
	static const char *
	promotion_attr_value(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *name)
	{
	char *attr_name = NULL;
	const char *attr_value = NULL;
	enum pcmk__rsc_node node_type = pcmk__rsc_node_assigned;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// Not assigned yet
	node_type = pcmk__rsc_node_current;
	}
	attr_name = pcmk_promotion_score_name(name);
	attr_value = pe__node_attribute_calculated(node, attr_name, rsc, node_type,
	false);
	free(attr_name);
	return attr_value;
	}

	/*!
	* \internal
	* \brief Get the promotion score for a clone instance on a node
	*
	* \param[in] rsc Promotable clone instance to get score for
	* \param[in] node Node to get score for
	* \param[out] is_default If non-NULL, will be set true if no score available
	*
	* \return Promotion score for \p rsc on \p node (or 0 if none)
	*/
	static int
	promotion_score(const pcmk_resource_t rsc, const pcmk_node_t node,
	bool *is_default)
	{
	char *name = NULL;
	const char *attr_value = NULL;

	if (is_default != NULL) {
	*is_default = true;
	}

	CRM_CHECK((rsc != NULL) && (node != NULL), return 0);

	/* If this is an instance of a cloned group, the promotion score is the sum
	* of all members' promotion scores.
	*/
	if (rsc->children != NULL) {
	int score = 0;

	for (const GList *iter = rsc->children;
	iter != NULL; iter = iter->next) {

	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	bool child_default = false;
	int child_score = promotion_score(child, node, &child_default);

	if (!child_default && (is_default != NULL)) {
	*is_default = false;
	}
	score += child_score;
	}
	return score;
	}

	if (!promotion_score_applies(rsc, node)) {
	return 0;
	}

	/* For the promotion score attribute name, use the name the resource is
	* known as in resource history, since that's what crm_attribute --promotion
	* would have used.
	*/
	name = (rsc->clone_name == NULL)? rsc->id : rsc->clone_name;

	attr_value = promotion_attr_value(rsc, node, name);
	if (attr_value != NULL) {
	pe_rsc_trace(rsc, "Promotion score for %s on %s = %s",
	name, pe__node_name(node), pcmk__s(attr_value, "(unset)"));
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	/* If we don't have any resource history yet, we won't have clone_name.
	* In that case, for anonymous clones, try the resource name without
	* any instance number.
	*/
	name = clone_strip(rsc->id);
	if (strcmp(rsc->id, name) != 0) {
	attr_value = promotion_attr_value(rsc, node, name);
	pe_rsc_trace(rsc, "Promotion score for %s on %s (for %s) = %s",
	name, pe__node_name(node), rsc->id,
	pcmk__s(attr_value, "(unset)"));
	}
	free(name);
	}

	if (attr_value == NULL) {
	return 0;
	}

	if (is_default != NULL) {
	*is_default = false;
	}
	return char2score(attr_value);
	}

	/*!
	* \internal
	* \brief Include promotion scores in instances' node scores and priorities
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__add_promotion_scores(pcmk_resource_t *rsc)
	{
	if (pe__set_clone_flag(rsc,
	pcmk__clone_promotion_added) == pcmk_rc_already) {
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	int score, new_score;

	g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, false, false)) {
	/* This node will never be promoted, so don't apply the
	* promotion score, as that may lead to clone shuffling.
	*/
	continue;
	}

	score = promotion_score(child_rsc, node, NULL);
	if (score > 0) {
	new_score = pcmk__add_scores(node->weight, score);
	if (new_score != node->weight) { // Could remain INFINITY
	node->weight = new_score;
	pe_rsc_trace(rsc,
	"Added %s promotion priority (%s) to score "
	"on %s (now %s)",
	child_rsc->id, pcmk_readable_score(score),
	pe__node_name(node),
	pcmk_readable_score(new_score));
	}
	}

	if (score > child_rsc->priority) {
	pe_rsc_trace(rsc,
	"Updating %s priority to promotion score (%d->%d)",
	child_rsc->id, child_rsc->priority, score);
	child_rsc->priority = score;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief If a resource's current role is started, change it to unpromoted
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_current_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->role == pcmk_role_started) {
	// Promotable clones should use unpromoted role instead of started
	rsc->role = pcmk_role_unpromoted;
	}
	g_list_foreach(rsc->children, set_current_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to unpromoted (or stopped if unassigned)
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_unpromoted(void data, void user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;
	GList *assigned = NULL;

	rsc->fns->location(rsc, &assigned, FALSE);
	if (assigned == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "stopped instance");
	} else {
	pe__set_next_role(rsc, pcmk_role_unpromoted, "unpromoted instance");
	g_list_free(assigned);
	}
	g_list_foreach(rsc->children, set_next_role_unpromoted, NULL);
	}

	/*!
	* \internal
	* \brief Set a resource's next role to promoted if not already set
	*
	* \param[in,out] data Resource to update
	* \param[in] user_data Ignored
	*/
	static void
	set_next_role_promoted(void *data, gpointer user_data)
	{
	pcmk_resource_t rsc = (pcmk_resource_t ) data;

	if (rsc->next_role == pcmk_role_unknown) {
	pe__set_next_role(rsc, pcmk_role_promoted, "promoted instance");
	}
	g_list_foreach(rsc->children, set_next_role_promoted, NULL);
	}

	/*!
	* \internal
	* \brief Show instance's promotion score on node where it will be active
	*
	* \param[in,out] instance Promotable clone instance to show
	*/
	static void
	show_promotion_score(pcmk_resource_t *instance)
	{
	pcmk_node_t *chosen = instance->fns->location(instance, NULL, FALSE);

	if (pcmk_is_set(instance->cluster->flags, pcmk_sched_output_scores)
	&& !pcmk__is_daemon && (instance->cluster->priv != NULL)) {

	pcmk__output_t *out = instance->cluster->priv;

	out->message(out, "promotion-score", instance, chosen,
	pcmk_readable_score(instance->sort_index));
	} else {
	pe_rsc_debug(pe__const_top_resource(instance, false),
	"%s promotion score on %s: sort=%s priority=%s",
	instance->id,
	((chosen == NULL)? "none" : pe__node_name(chosen)),
	pcmk_readable_score(instance->sort_index),
	pcmk_readable_score(instance->priority));
	}
	}

	/*!
	* \internal
	* \brief Set a clone instance's promotion priority
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in] user_data Instance's parent clone
	*/
	static void
	set_instance_priority(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	const pcmk_resource_t clone = (const pcmk_resource_t ) user_data;
	const pcmk_node_t *chosen = NULL;
	enum rsc_role_e next_role = pcmk_role_unknown;
	GList *list = NULL;

	pe_rsc_trace(clone, "Assigning priority for %s: %s", instance->id,
	role2text(instance->next_role));

	if (instance->fns->state(instance, TRUE) == pcmk_role_started) {
	set_current_role_unpromoted(instance, NULL);
	}

	// Only an instance that will be active can be promoted
	chosen = instance->fns->location(instance, &list, FALSE);
	if (pcmk__list_of_multiple(list)) {
	pcmk__config_err("Cannot promote non-colocated child %s",
	instance->id);
	}
	g_list_free(list);
	if (chosen == NULL) {
	return;
	}

	next_role = instance->fns->state(instance, FALSE);
	switch (next_role) {
	case pcmk_role_started:
	case pcmk_role_unknown:
	// Set instance priority to its promotion score (or -1 if none)
	{
	bool is_default = false;

	instance->priority = promotion_score(instance, chosen,
	&is_default);
	if (is_default) {
	/*
	* Default to -1 if no value is set. This allows
	* instances eligible for promotion to be specified
	* based solely on rsc_location constraints, but
	* prevents any instance from being promoted if neither
	* a constraint nor a promotion score is present
	*/
	instance->priority = -1;
	}
	}
	break;

	case pcmk_role_unpromoted:
	case pcmk_role_stopped:
	// Instance can't be promoted
	instance->priority = -INFINITY;
	break;

	case pcmk_role_promoted:
	// Nothing needed (re-creating actions after scheduling fencing)
	break;

	default:
	CRM_CHECK(FALSE, crm_err("Unknown resource role %d for %s",
	next_role, instance->id));
	}

	// Add relevant location constraint scores for promoted role
	apply_promoted_locations(instance, instance->rsc_location, chosen);
	apply_promoted_locations(instance, clone->rsc_location, chosen);

	// Consider instance's role-based colocations with other resources
	list = pcmk__this_with_colocations(instance);
	for (GList *iter = list; iter != NULL; iter = iter->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) iter->data;

	instance->cmds->apply_coloc_score(instance, cons->primary, cons, true);
	}
	g_list_free(list);

	instance->sort_index = instance->priority;
	if (next_role == pcmk_role_promoted) {
	instance->sort_index = INFINITY;
	}
	pe_rsc_trace(clone, "Assigning %s priority = %d",
	instance->id, instance->priority);
	}

	/*!
	* \internal
	* \brief Set a promotable clone instance's role
	*
	* \param[in,out] data Promotable clone instance to update
	* \param[in,out] user_data Pointer to count of instances chosen for promotion
	*/
	static void
	set_instance_role(gpointer data, gpointer user_data)
	{
	pcmk_resource_t instance = (pcmk_resource_t ) data;
	int count = (int ) user_data;

	const pcmk_resource_t *clone = pe__const_top_resource(instance, false);
	pcmk_node_t *chosen = NULL;

	show_promotion_score(instance);

	if (instance->sort_index < 0) {
	pe_rsc_trace(clone, "Not supposed to promote instance %s",
	instance->id);

	} else if ((*count < pe__clone_promoted_max(instance))
	\|\| !pcmk_is_set(clone->flags, pcmk_rsc_managed)) {
	chosen = node_to_be_promoted_on(instance);
	}

	if (chosen == NULL) {
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	if ((instance->role < pcmk_role_promoted)
	&& !pcmk_is_set(instance->cluster->flags, pcmk_sched_quorate)
	&& (instance->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Clone instance %s cannot be promoted without quorum",
	instance->id);
	set_next_role_unpromoted(instance, NULL);
	return;
	}

	chosen->count++;
	pe_rsc_info(clone, "Choosing %s (%s) on %s for promotion",
	instance->id, role2text(instance->role),
	pe__node_name(chosen));
	set_next_role_promoted(instance, NULL);
	(*count)++;
	}

	/*!
	* \internal
	* \brief Set roles for all instances of a promotable clone
	*
	* \param[in,out] rsc Promotable clone resource to update
	*/
	void
	pcmk__set_instance_roles(pcmk_resource_t *rsc)
	{
	int promoted = 0;
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	// Repurpose count to track the number of promoted instances assigned
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	node->count = 0;
	}

	// Set instances' promotion priorities and sort by highest priority first
	g_list_foreach(rsc->children, set_instance_priority, rsc);
	sort_promotable_instances(rsc);

	// Choose the first N eligible instances to be promoted
	g_list_foreach(rsc->children, set_instance_role, &promoted);
	pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
	rsc->id, promoted, pe__clone_promoted_max(rsc));
	}

	/*!
	*
	* \internal
	* \brief Create actions for promotable clone instances
	*
	* \param[in,out] clone Promotable clone to create actions for
	* \param[out] any_promoting Will be set true if any instance is promoting
	* \param[out] any_demoting Will be set true if any instance is demoting
	*/
	static void
	create_promotable_instance_actions(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting)
	{
	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->cmds->create_actions(instance);
	check_for_role_change(instance, any_demoting, any_promoting);
	}
	}

	/*!
	* \internal
	* \brief Reset each promotable instance's resource priority
	*
	* Reset the priority of each instance of a promotable clone to the clone's
	* priority (after promotion actions are scheduled, when instance priorities
	* were repurposed as promotion scores).
	*
	* \param[in,out] clone Promotable clone to reset
	*/
	static void
	reset_instance_priorities(pcmk_resource_t *clone)
	{
	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	instance->priority = clone->priority;
	}
	}

	/*!
	* \internal
	* \brief Create actions specific to promotable clones
	*
	* \param[in,out] clone Promotable clone to create actions for
	*/
	void
	pcmk__create_promotable_actions(pcmk_resource_t *clone)
	{
	bool any_promoting = false;
	bool any_demoting = false;

	// Create actions for each clone instance individually
	create_promotable_instance_actions(clone, &any_promoting, &any_demoting);

	// Create pseudo-actions for clone as a whole
	pe__create_promotable_pseudo_ops(clone, any_promoting, any_demoting);

	// Undo our temporary repurposing of resource priority for instances
	reset_instance_priorities(clone);
	}

	/*!
	* \internal
	* \brief Create internal orderings for a promotable clone's instances
	*
	* \param[in,out] clone Promotable clone instance to order
	*/
	void
	pcmk__order_promotable_instances(pcmk_resource_t *clone)
	{
	pcmk_resource_t *previous = NULL; // Needed for ordered clones

	pcmk__promotable_restart_ordering(clone);

	for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;

	// Demote before promote
	pcmk__order_resource_actions(instance, PCMK_ACTION_DEMOTE,
	instance, PCMK_ACTION_PROMOTE,
	pcmk__ar_ordered);

	order_instance_promotion(clone, instance, previous);
	order_instance_demotion(clone, instance, previous);
	previous = instance;
	}
	}

	/*!
	* \internal
	* \brief Update dependent's allowed nodes for colocation with promotable
	*
	* \param[in,out] dependent Dependent resource to update
	* \param[in] primary Primary resource
	* \param[in] primary_node Node where an instance of the primary will be
	* \param[in] colocation Colocation constraint to apply
	*/
	static void
	update_dependent_allowed_nodes(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk_node_t *primary_node,
	const pcmk__colocation_t *colocation)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;
	const char *primary_value = NULL;
	const char *attr = colocation->node_attribute;

	if (colocation->score >= INFINITY) {
	return; // Colocation is mandatory, so allowed node scores don't matter
	}

	primary_value = pcmk__colocation_node_attr(primary_node, attr, primary);

	pe_rsc_trace(colocation->primary,
	"Applying %s (%s with %s on %s by %s @%d) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, pe__node_name(primary_node), attr,
	colocation->score, dependent->id);

	g_hash_table_iter_init(&iter, dependent->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	const char *dependent_value = pcmk__colocation_node_attr(node, attr,
	dependent);

	if (pcmk__str_eq(primary_value, dependent_value, pcmk__str_casei)) {
	node->weight = pcmk__add_scores(node->weight, colocation->score);
	pe_rsc_trace(colocation->primary,
	"Added %s score (%s) to %s (now %s)",
	colocation->id, pcmk_readable_score(colocation->score),
	pe__node_name(node),
	pcmk_readable_score(node->weight));
	}
	}
	}

	/*!
	* \brief Update dependent for a colocation with a promotable clone
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_dependent_with_promotable(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	GList *affected_nodes = NULL;

	/* Build a list of all nodes where an instance of the primary will be, and
	* (for optional colocations) update the dependent's allowed node scores for
	* each one.
	*/
	for (GList *iter = primary->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t instance = (pcmk_resource_t ) iter->data;
	pcmk_node_t *node = instance->fns->location(instance, NULL, FALSE);

	if (node == NULL) {
	continue;
	}
	if (instance->fns->state(instance, FALSE) == colocation->primary_role) {
	update_dependent_allowed_nodes(dependent, primary, node,
	colocation);
	affected_nodes = g_list_prepend(affected_nodes, node);
	}
	}

	/* For mandatory colocations, add the primary's node score to the
	* dependent's node score for each affected node, and ban the dependent
	* from all other nodes.
	*
	* However, skip this for promoted-with-promoted colocations, otherwise
	* inactive dependent instances can't start (in the unpromoted role).
	*/
	if ((colocation->score >= INFINITY)
	&& ((colocation->dependent_role != pcmk_role_promoted)
	\|\| (colocation->primary_role != pcmk_role_promoted))) {

	pe_rsc_trace(colocation->primary,
	"Applying %s (mandatory %s with %s) to %s",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	pcmk__colocation_intersect_nodes(dependent, primary, colocation,
	affected_nodes, true);
	}
	g_list_free(affected_nodes);
	}

	/*!
	* \internal
	* \brief Update dependent priority for colocation with promotable
	*
	* \param[in] primary Primary resource in the colocation
	* \param[in,out] dependent Dependent resource in the colocation
	* \param[in] colocation Colocation constraint to apply
	*/
	void
	pcmk__update_promotable_dependent_priority(const pcmk_resource_t *primary,
	pcmk_resource_t *dependent,
	const pcmk__colocation_t *colocation)
	{
	pcmk_resource_t *primary_instance = NULL;

	// Look for a primary instance where dependent will be
	primary_instance = pcmk__find_compatible_instance(dependent, primary,
	colocation->primary_role,
	false);

	if (primary_instance != NULL) {
	// Add primary instance's priority to dependent's
	int new_priority = pcmk__add_scores(dependent->priority,
	colocation->score);

	pe_rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s priority (%s + %s = %s)",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id,
	pcmk_readable_score(dependent->priority),
	pcmk_readable_score(colocation->score),
	pcmk_readable_score(new_priority));
	dependent->priority = new_priority;

	} else if (colocation->score >= INFINITY) {
	// Mandatory colocation, but primary won't be here
	pe_rsc_trace(colocation->primary,
	"Applying %s (%s with %s) to %s: can't be promoted",
	colocation->id, colocation->dependent->id,
	colocation->primary->id, dependent->id);
	dependent->priority = -INFINITY;
	}
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index 982b578102..039bfd4f9e 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3164 +1,3164 @@
	/*
	* Copyright 2019-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;
	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	"shutdown", PCMK_NODE_ATTR_TERMINATE, "standby", "#", \
	NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, "type");
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(node, rsc->children, scheduler, attrname,
	expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, "name");

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {
	const char *task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(rsc_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;
	pcmk_create_xml_text_node(node, (const char ) key, (const char ) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
	scheduler->input, LOG_DEBUG);
	return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair("interval", interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE, &epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_EXEC, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_QUEUE, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
	failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK__LAST_FAILURE_PREFIX, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: migration-threshold=%d%s%s",
	rsc_id, rsc->migration_threshold, failcount_s,
	lastfail_s? lastfail_s : "");
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online && node->details->expected_up
	&& !pe__is_guest_or_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->details->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static char *
	formatted_xml_buf(const pcmk_resource_t *rsc, bool raw)
	{
	if (raw) {
	return dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
	} else {
	return dump_xml_formatted(rsc->xml);
	}
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	XML_ATTR_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	XML_CIB_ATTR_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_ORIG);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	XML_ATTR_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	XML_CIB_ATTR_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_ORIG);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pe__is_guest_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;
	const pcmk_node_t *host_node = pe__current_node(container);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = malloc(name_len);
	CRM_ASSERT(node_name != NULL);
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...)
	{
	xmlNodePtr xml_node = NULL;
	va_list args;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = create_xml_node(xml_node, tag_name);

	va_start(args, pairs_count);
	while(pairs_count--) {
	const char param_name = va_arg(args, const char );
	const char param_value = va_arg(args, const char );
	if (param_name && param_value) {
	crm_xml_add(xml_node, param_name, param_value);
	}
	};
	va_end(args);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	#ifdef PCMK__COMPAT_2_0
	return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
	#else
	return "in " PCMK__ROLE_PROMOTED " role ";
	#endif
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	char *weight_s = pcmk__itoa(pe_node->weight);

	pcmk__output_create_xml_node(out, "ban",
	"id", location->id,
	"resource", location->rsc->id,
	"node", pe_node->details->uname,
	"weight", weight_s,
	"promoted-only", promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	"master_only", promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	- for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
	+ for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *nnodes_str = crm_strdup_printf("%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
	free(nnodes_str);

	if (ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");

	s = crm_strdup_printf(", %d ", nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else if (ndisabled && !nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
	} else if (!ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else {
	char *s = crm_strdup_printf("%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured", NULL);

	char *s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, "number", s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, "number", s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, "disabled", s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, "blocked", s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");

	if (dc) {
	char *buf = crm_strdup_printf("%s (version %s) -", dc_name,
	dc_version_s ? dc_version_s : "unknown");
	pcmk_create_html_node(node, "span", NULL, NULL, buf);
	free(buf);

	if (mixed_version) {
	pcmk_create_html_node(node, "span", NULL, "warning",
	" MIXED-VERSION");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " partition");
	if (crm_is_true(quorum)) {
	pcmk_create_html_node(node, "span", NULL, NULL, " with");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", " WITHOUT");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " quorum");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", "true",
	"version", dc_version_s ? dc_version_s : "",
	"name", dc->details->uname,
	"id", dc->details->id,
	"with_quorum", pcmk__btoa(crm_is_true(quorum)),
	"mixed_version", pcmk__btoa(mixed_version),
	NULL);
	} else {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", "false",
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will not attempt to start, stop, or recover services)");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "STOPPED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will keep all resources stopped)");
	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	#define bv(flag) pcmk__btoa(pcmk_is_set(scheduler->flags, (flag)))

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *no_quorum_policy = NULL;
	char *stonith_timeout_str = pcmk__itoa(scheduler->stonith_timeout);
	char priority_fencing_delay_str = pcmk__itoa(scheduler->priority_fencing_delay 1000);

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	no_quorum_policy = "freeze";
	break;

	case pcmk_no_quorum_stop:
	no_quorum_policy = "stop";
	break;

	case pcmk_no_quorum_demote:
	no_quorum_policy = "demote";
	break;

	case pcmk_no_quorum_ignore:
	no_quorum_policy = "ignore";
	break;

	case pcmk_no_quorum_fence:
	no_quorum_policy = "suicide";
	break;
	}

	pcmk__output_create_xml_node(out, "cluster_options",
	"stonith-enabled",
	bv(pcmk_sched_fencing_enabled),
	"symmetric-cluster",
	bv(pcmk_sched_symmetric_cluster),
	"no-quorum-policy", no_quorum_policy,
	"maintenance-mode",
	bv(pcmk_sched_in_maintenance),
	"stop-all-resources", bv(pcmk_sched_stop_all),
	"stonith-timeout-ms", stonith_timeout_str,
	"priority-fencing-delay-ms", priority_fencing_delay_str,
	NULL);
	free(stonith_timeout_str);
	free(priority_fencing_delay_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
	pcmk_create_html_node(node, "span", NULL, NULL, stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	pcmk_create_html_node(node, "span", NULL, NULL, " (");
	pcmk_create_html_node(node, "span", NULL, NULL,
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	pcmk_create_html_node(node, "span", NULL, NULL, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, "stack",
	"type", stack_s,
	"pacemakerd-state", state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, time_s);

	if (our_nodename != NULL) {
	pcmk_create_html_node(updated_node, "span", NULL, NULL, " on ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, our_nodename);
	}

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
	pcmk_create_html_node(changed_node, "span", NULL, NULL, time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, "last_update",
	"time", time_s,
	"origin", our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, "last_change",
	"time", last_written ? last_written : "",
	"user", user ? user : "",
	"client", client ? client : "",
	"origin", origin ? origin : "",
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
	rsc_id = strdup("unknown resource");
	task = strdup("unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	const char *queue_time = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " XML_RSC_OP_LAST_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, XML_ATTR_UNAME);
	const char *exit_reason = crm_element_value(xml_op,
	XML_LRM_ATTR_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *op_key_name = "op_key";
	int rc;
	int status;
	const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);

	time_t epoch = 0;
	char *rc_s = NULL;
	char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
	xmlNodePtr node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	rc_s = pcmk__itoa(rc);
	if (crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY) == NULL) {
	op_key_name = "id";
	}
	node = pcmk__output_create_xml_node(out, "failure",
	op_key_name, op_key,
	"node", crm_element_value(xml_op, XML_ATTR_UNAME),
	"exitstatus", services_ocf_exitcode_str(rc),
	"exitreason", pcmk__s(reason_s, ""),
	"exitcode", rc_s,
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"status", pcmk_exec_status_str(status),
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE, rc_change,
	"queued", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	"exec", crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	"interval", interval_ms_s,
	"task", crm_element_value(xml_op, XML_LRM_ATTR_TASK),
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	free(reason_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xml_first_child(scheduler->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, XML_ATTR_UNAME), only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pe__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);

	// Cluster membership
	if (node->details->online) {
	pcmk_create_html_node(parent, "span", NULL, "online", " online");
	} else {
	pcmk_create_html_node(parent, "span", NULL, "offline", " OFFLINE");
	}

	// Standby mode
	if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail,"
	" with active resources)");
	} else if (node->details->standby_onfail) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail)");
	} else if (node->details->standby && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby, with active resources)");
	} else if (node->details->standby) {
	pcmk_create_html_node(parent, "span", NULL, "standby", " (in standby)");
	}

	// Maintenance mode
	if (node->details->maintenance) {
	pcmk_create_html_node(parent, "span", NULL, "maint",
	" (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_red",
	" (health is RED)");
	} else if (health == 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_yellow",
	" (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	char *buf = crm_strdup_printf(", feature set %s", feature_set);
	pcmk_create_html_node(parent, "span", NULL, NULL, buf);
	free(buf);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNodePtr item_node;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	char *buf = crm_strdup_printf("%s:", node_name);

	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	status_node(node, item_node, show_opts);

	free(buf);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	return "standby (on-fail)";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	return "standby (with active resources)";
	} else {
	return "standby";
	}
	} else {
	return "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pe__is_guest_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pe__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *node_type = "unknown";
	char *length_s = pcmk__itoa(g_list_length(node->details->running_rsc));
	int health = pe__node_health(node);
	const char *health_s = NULL;
	const char *feature_set;

	switch (node->details->type) {
	case pcmk_node_variant_cluster:
	node_type = "member";
	break;
	case pcmk_node_variant_remote:
	node_type = "remote";
	break;
	case node_ping:
	node_type = "ping";
	break;
	}

	if (health < 0) {
	health_s = "red";
	} else if (health == 0) {
	health_s = "yellow";
	} else {
	health_s = "green";
	}

	feature_set = get_node_feature_set(node);

	pe__name_and_nvpairs_xml(out, true, "node", 15,
	"name", node->details->uname,
	"id", node->details->id,
	"online", pcmk__btoa(node->details->online),
	"standby", pcmk__btoa(node->details->standby),
	"standby_onfail", pcmk__btoa(node->details->standby_onfail),
	"maintenance", pcmk__btoa(node->details->maintenance),
	"pending", pcmk__btoa(node->details->pending),
	"unclean", pcmk__btoa(node->details->unclean),
	"health", health_s,
	"feature_set", feature_set,
	"shutdown", pcmk__btoa(node->details->shutdown),
	"expected_up", pcmk__btoa(node->details->expected_up),
	"is_dc", pcmk__btoa(node->details->is_dc),
	"resources_running", length_s,
	"type", node_type);

	if (pe__is_guest_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, "id_as_resource", node->details->remote_rsc->container->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}
	}

	free(length_s);

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, "node",
	"name", node->details->uname,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;
	char *s = crm_strdup_printf("%s: %s", name, value);
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	pcmk_create_html_node(item_node, "span", NULL, NULL, s);
	free(s);

	if (v <= 0) {
	pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
	} else if (v < expected_score) {
	char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	free(buf);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pe__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pe__xe_history_key(xml_op),
	crm_element_value(xml_op, XML_ATTR_UNAME),
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	crm_element_value(xml_op, XML_LRM_ATTR_RC),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);
	node = pcmk__output_create_xml_node(out, "operation",
	"op", pe__xe_history_key(xml_op),
	"node", crm_element_value(xml_op, XML_ATTR_UNAME),
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"rc", crm_element_value(xml_op, XML_LRM_ATTR_RC),
	"status", pcmk_exec_status_str(status),
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	char *agent_tuple = NULL;

	agent_tuple = crm_strdup_printf("%s:%s:%s", class,
	pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider) ? crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER) : "",
	kind);

	pcmk__xe_set_props(node, "rsc", rsc_printable_id(rsc),
	"agent", agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	XML_RSC_OP_T_EXEC, crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "attribute",
	"name", name,
	"value", value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, "expected", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pe_node_attribute_raw(node, name);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pe__node_name(node));

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "capacity",
	"node", node->details->uname,
	"comment", comment,
	NULL);
	g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNodePtr", "GList ", "GList ", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = first_named_child(lrm_rsc, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (parent->variant == pcmk_rsc_variant_group) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, XML_ATTR_ID));

	if (op_list == NULL) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (node->details->standby_onfail && node->details->online)
	\|\| node->details->standby \|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pe__is_guest_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pe__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pe__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pe__is_guest_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	out->begin_list(out, NULL, NULL, "nodes");
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	for (node_state = first_named_child(cib_status, XML_CIB_TAG_STATE);
	node_state != NULL; node_state = crm_next_same_xml(node_state)) {
	pcmk_node_t *node = pe_find_node_id(scheduler->nodes, ID(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "node_weight",
	"function", prefix,
	"node", uname,
	"score", score,
	NULL);

	if (rsc) {
	crm_xml_add(node, "id", rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *rc_s = pcmk__itoa(rc);
	xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history",
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"task", task,
	"rc", rc_s,
	"rc_text", services_ocf_exitcode_str(rc),
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, "interval", s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, XML_RSC_OP_LAST_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_EXEC, s);
	free(s);
	}
	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_QUEUE, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id,
	chosen? chosen->details->uname : "none",
	score);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "promotion_score",
	"id", child_rsc->id,
	"score", score,
	NULL);

	if (chosen) {
	crm_xml_add(node, "node", chosen->details->uname);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	pcmk__formatted_printf(out, "Resource XML:\n");
	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history",
	"id", rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, "orphan", true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->migration_threshold);

	pcmk__xe_set_props(node, "orphan", "false",
	"migration-threshold", migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK__FAIL_COUNT_PREFIX, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK__LAST_FAILURE_PREFIX, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& (rsc->variant == pcmk_rsc_variant_primitive)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
	only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(xml_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pe__node_name(node));

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "utilization",
	"resource", rsc->id,
	"node", node->details->uname,
	"function", fn,
	NULL);
	g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_html(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s:\t%s%s %s=\"%s\"", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_text(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, ticket->id, "%s%s %s=\"%s\"",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, ticket->id, "%s%s",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "ticket",
	"id", ticket->id,
	"status", ticket->granted ? "granted" : "revoked",
	"standby", pcmk__btoa(ticket->standby),
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, "last-granted", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "pcmk_scheduler_t *", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	bool print_spacer = va_arg(args, int);

	GHashTableIter iter;
	gpointer key, value;

	if (g_hash_table_size(scheduler->tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, scheduler->tickets);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_text },
	{ "ticket", "html", ticket_html },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index a8ae199b47..b7ce77f715 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,483 +1,483 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler != NULL) {
	set_working_set_defaults(scheduler);
	}
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	scheduler->priv = NULL;
	free(scheduler);
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->failed != NULL) {
	free_xml(scheduler->failed);
	}
	scheduler->failed = create_xml_node(NULL, "failed-ops");

	if (scheduler->now == NULL) {
	scheduler->now = crm_time_new(NULL);
	}

	if (scheduler->dc_uuid == NULL) {
	scheduler->dc_uuid = crm_element_value_copy(scheduler->input,
	XML_ATTR_DC_UUID);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, XML_ATTR_HAVE_QUORUM)) {
	pe__set_working_set_flags(scheduler, pcmk_sched_quorate);
	} else {
	pe__clear_working_set_flags(scheduler, pcmk_sched_quorate);
	}

	scheduler->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
	scheduler->input, LOG_NEVER);
	scheduler->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
	scheduler->input, LOG_NEVER);

	section = get_xpath_object("//" XML_CIB_TAG_CRMCONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_location_only\|pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	crm_warn("Fencing and resource management disabled due to lack of quorum");
	}

	section = get_xpath_object("//" XML_CIB_TAG_NODES, scheduler->input,
	LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_TAGS, scheduler->input,
	LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	section = get_xpath_object("//"XML_CIB_TAG_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_no_counts)) {
	for (GList *item = scheduler->resources; item != NULL;
	item = item->next) {
	((pcmk_resource_t *) (item->data))->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	pe__set_working_set_flags(scheduler, pcmk_sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->details->uname for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
	"(guest or remote)" : pe__node_name(node)));

	if (node->details->attrs != NULL) {
	g_hash_table_destroy(node->details->attrs);
	}
	if (node->details->utilization != NULL) {
	g_hash_table_destroy(node->details->utilization);
	}
	if (node->details->digest_cache != NULL) {
	g_hash_table_destroy(node->details->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->details->allocated_rsc);
	free(node->details);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pe__ordering_t *order = iterator->data;

	iterator = iterator->next;

	free(order->lh_action_task);
	free(order->rh_action_task);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	- g_list_free_full(cons->node_list_rh, free);
	+ g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	pe__clear_working_set_flags(scheduler, pcmk_sched_have_status);
	if (scheduler->config_hash != NULL) {
	g_hash_table_destroy(scheduler->config_hash);
	}

	if (scheduler->singletons != NULL) {
	g_hash_table_destroy(scheduler->singletons);
	}

	if (scheduler->tickets) {
	g_hash_table_destroy(scheduler->tickets);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	free(scheduler->dc_uuid);

	crm_trace("deleting resources");
	pe_free_resources(scheduler->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	free_xml(scheduler->graph);
	crm_time_free(scheduler->now);
	free_xml(scheduler->input);
	free_xml(scheduler->failed);

	set_working_set_defaults(scheduler);

	CRM_CHECK(scheduler->ordering_constraints == NULL,;
	);
	CRM_CHECK(scheduler->placement_constraints == NULL,;
	);
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->ordering_constraints));
	pe__free_ordering(scheduler->ordering_constraints);
	scheduler->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->placement_constraints));
	pe__free_location(scheduler->placement_constraints);
	scheduler->placement_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->colocation_constraints));
	g_list_free_full(scheduler->colocation_constraints, free);
	scheduler->colocation_constraints = NULL;

	crm_trace("Deleting %d ticket constraints",
	g_list_length(scheduler->ticket_constraints));
	g_list_free_full(scheduler->ticket_constraints, free);
	scheduler->ticket_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	void *priv = scheduler->priv;

	memset(scheduler, 0, sizeof(pcmk_scheduler_t));

	scheduler->priv = priv;
	scheduler->order_id = 1;
	scheduler->action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;

	scheduler->flags = 0x0ULL;

	pe__set_working_set_flags(scheduler,
	pcmk_sched_symmetric_cluster
	\|pcmk_sched_stop_removed_resources
	\|pcmk_sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, "true")) {
	pe__set_working_set_flags(scheduler, pcmk_sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;

	pcmk_resource_t *match =
	parent->fns->find_rsc(parent, id, NULL, flags);
	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pe_find_node(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->details->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (pcmk__str_eq(node->details->uname, node_name, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 8:14 PM (6 h, 25 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665703
Default Alt Text: (324 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions