No OneTemporary
Actions

Size

233 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/fencing/st_output.c b/lib/fencing/st_output.c
	index 145dd141e4..e1f4830fac 100644
	--- a/lib/fencing/st_output.c
	+++ b/lib/fencing/st_output.c
	@@ -1,460 +1,460 @@
	/*
	* Copyright 2019-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <stdarg.h>

	#include <crm/stonith-ng.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/util.h>
	#include <crm/common/xml.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/fencing/internal.h>
	#include <crm/pengine/internal.h>

	static char *
	time_t_string(time_t when) {
	crm_time_t *crm_when = crm_time_new(NULL);
	char *buf = NULL;

	crm_time_set_timet(crm_when, &when);
	buf = crm_time_as_string(crm_when, crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	crm_time_free(crm_when);
	return buf;
	}

	-PCMK__OUTPUT_ARGS("failed-fencing-history", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	+PCMK__OUTPUT_ARGS("failed-fencing-list", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	int
	stonith__failed_history(pcmk__output_t *out, va_list args) {
	stonith_history_t history = va_arg(args, stonith_history_t );
	GList only_node = va_arg(args, GList );
	gboolean full_history = va_arg(args, gboolean);
	gboolean print_spacer = va_arg(args, gboolean);

	int rc = pcmk_rc_no_output;

	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (hp->state != st_failed) {
	continue;
	}

	if (!pcmk__str_in_list(only_node, hp->target)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Fencing Actions");
	out->message(out, "stonith-event", hp, full_history, stonith__later_succeeded(hp, history));
	out->increment_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	-PCMK__OUTPUT_ARGS("fencing-history", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	+PCMK__OUTPUT_ARGS("fencing-list", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	int
	stonith__history(pcmk__output_t *out, va_list args) {
	stonith_history_t history = va_arg(args, stonith_history_t );
	GList only_node = va_arg(args, GList );
	gboolean full_history = va_arg(args, gboolean);
	gboolean print_spacer = va_arg(args, gboolean);

	int rc = pcmk_rc_no_output;

	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (!pcmk__str_in_list(only_node, hp->target)) {
	continue;
	}

	if (hp->state != st_failed) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Fencing History");
	out->message(out, "stonith-event", hp, full_history, stonith__later_succeeded(hp, history));
	out->increment_list(out);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	-PCMK__OUTPUT_ARGS("full-fencing-history", "crm_exit_t", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	+PCMK__OUTPUT_ARGS("full-fencing-list", "crm_exit_t", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	int
	stonith__full_history(pcmk__output_t *out, va_list args) {
	crm_exit_t history_rc G_GNUC_UNUSED = va_arg(args, crm_exit_t);
	stonith_history_t history = va_arg(args, stonith_history_t );
	GList only_node = va_arg(args, GList );
	gboolean full_history = va_arg(args, gboolean);
	gboolean print_spacer = va_arg(args, gboolean);

	int rc = pcmk_rc_no_output;

	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (!pcmk__str_in_list(only_node, hp->target)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Fencing History");
	out->message(out, "stonith-event", hp, full_history, stonith__later_succeeded(hp, history));
	out->increment_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	-PCMK__OUTPUT_ARGS("full-fencing-history", "crm_exit_t", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	+PCMK__OUTPUT_ARGS("full-fencing-list", "crm_exit_t", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	int
	stonith__full_history_xml(pcmk__output_t *out, va_list args) {
	crm_exit_t history_rc = va_arg(args, crm_exit_t);
	stonith_history_t history = va_arg(args, stonith_history_t );
	GList only_node = va_arg(args, GList );
	gboolean full_history = va_arg(args, gboolean);
	gboolean print_spacer G_GNUC_UNUSED = va_arg(args, gboolean);

	int rc = pcmk_rc_no_output;

	if (history_rc == 0) {
	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (!pcmk__str_in_list(only_node, hp->target)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Fencing History");
	out->message(out, "stonith-event", hp, full_history, stonith__later_succeeded(hp, history));
	out->increment_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	} else {
	char *rc_s = crm_itoa(history_rc);

	pcmk__output_create_xml_node(out, "fence_history",
	"status", rc_s,
	NULL);
	free(rc_s);

	rc = pcmk_rc_ok;
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("last-fenced", "const char *", "time_t")
	int
	stonith__last_fenced_html(pcmk__output_t *out, va_list args) {
	const char target = va_arg(args, const char );
	time_t when = va_arg(args, time_t);

	if (when) {
	char *buf = crm_strdup_printf("Node %s last fenced at: %s", target, ctime(&when));
	pcmk__output_create_html_node(out, "div", NULL, NULL, buf);
	free(buf);
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("last-fenced", "const char *", "time_t")
	int
	stonith__last_fenced_text(pcmk__output_t *out, va_list args) {
	const char target = va_arg(args, const char );
	time_t when = va_arg(args, time_t);

	if (when) {
	pcmk__indented_printf(out, "Node %s last fenced at: %s", target, ctime(&when));
	} else {
	pcmk__indented_printf(out, "Node %s has never been fenced\n", target);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("last-fenced", "const char *", "time_t")
	int
	stonith__last_fenced_xml(pcmk__output_t *out, va_list args) {
	const char target = va_arg(args, const char );
	time_t when = va_arg(args, time_t);

	if (when) {
	char *buf = time_t_string(when);

	pcmk__output_create_xml_node(out, "last-fenced",
	"target", target,
	"when", buf,
	NULL);

	free(buf);
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	-PCMK__OUTPUT_ARGS("pending-fencing-actions", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	+PCMK__OUTPUT_ARGS("pending-fencing-list", "stonith_history_t ", "GList ", "gboolean", "gboolean")
	int
	stonith__pending_actions(pcmk__output_t *out, va_list args) {
	stonith_history_t history = va_arg(args, stonith_history_t );
	GList only_node = va_arg(args, GList );
	gboolean full_history = va_arg(args, gboolean);
	gboolean print_spacer = va_arg(args, gboolean);

	int rc = pcmk_rc_no_output;

	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (!pcmk__str_in_list(only_node, hp->target)) {
	continue;
	}

	/* Skip the rest of the history after we see a failed/done action */
	if ((hp->state == st_failed) \|\| (hp->state == st_done)) {
	break;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Pending Fencing Actions");
	out->message(out, "stonith-event", hp, full_history, stonith__later_succeeded(hp, history));
	out->increment_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("stonith-event", "stonith_history_t *", "gboolean", "gboolean")
	int
	stonith__event_html(pcmk__output_t *out, va_list args) {
	stonith_history_t event = va_arg(args, stonith_history_t );
	gboolean full_history = va_arg(args, gboolean);
	gboolean later_succeeded = va_arg(args, gboolean);

	switch(event->state) {
	case st_done: {
	char *completed_s = time_t_string(event->completed);

	out->list_item(out, "successful-stonith-event",
	"%s of %s successful: delegate=%s, client=%s, origin=%s, %s='%s'",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-successful",
	completed_s);
	free(completed_s);
	break;
	}

	case st_failed: {
	char *failed_s = time_t_string(event->completed);

	out->list_item(out, "failed-stonith-event",
	"%s of %s failed : delegate=%s, client=%s, origin=%s, %s='%s' %s",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-failed",
	failed_s,
	later_succeeded ? "(a later attempt succeeded)" : "");
	free(failed_s);
	break;
	}

	default:
	out->list_item(out, "pending-stonith-event",
	"%s of %s pending: client=%s, origin=%s",
	stonith_action_str(event->action), event->target,
	event->client, event->origin);
	break;
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("stonith-event", "stonith_history_t *", "gboolean", "gboolean")
	int
	stonith__event_text(pcmk__output_t *out, va_list args) {
	stonith_history_t event = va_arg(args, stonith_history_t );
	gboolean full_history = va_arg(args, gboolean);
	gboolean later_succeeded = va_arg(args, gboolean);

	char *buf = time_t_string(event->completed);

	switch (event->state) {
	case st_failed:
	pcmk__indented_printf(out, "%s of %s failed: delegate=%s, client=%s, origin=%s, %s='%s' %s\n",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-failed", buf,
	later_succeeded ? "(a later attempt succeeded)" : "");
	break;

	case st_done:
	pcmk__indented_printf(out, "%s of %s successful: delegate=%s, client=%s, origin=%s, %s='%s'\n",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-successful", buf);
	break;

	default:
	pcmk__indented_printf(out, "%s of %s pending: client=%s, origin=%s\n",
	stonith_action_str(event->action), event->target,
	event->client, event->origin);
	break;
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("stonith-event", "stonith_history_t *", "gboolean", "gboolean")
	int
	stonith__event_xml(pcmk__output_t *out, va_list args) {
	stonith_history_t event = va_arg(args, stonith_history_t );
	gboolean full_history G_GNUC_UNUSED = va_arg(args, gboolean);
	gboolean later_succeeded G_GNUC_UNUSED = va_arg(args, gboolean);

	char *buf = NULL;

	xmlNodePtr node = pcmk__output_create_xml_node(out, "fence_event",
	"action", event->action,
	"target", event->target,
	"client", event->client,
	"origin", event->origin,
	NULL);

	switch (event->state) {
	case st_failed:
	crm_xml_add(node, "status", "failed");
	break;

	case st_done:
	crm_xml_add(node, "status", "success");
	break;

	default: {
	char *state = crm_itoa(event->state);
	pcmk__xe_set_props(node, "status", "pending",
	"extended-status", state,
	NULL);
	free(state);
	break;
	}
	}

	if (event->delegate != NULL) {
	crm_xml_add(node, "delegate", event->delegate);
	}

	if (event->state == st_failed \|\| event->state == st_done) {
	buf = time_t_string(event->completed);
	crm_xml_add(node, "completed", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("validate", "const char ", "const char ", "char ", "char ", "int")
	int
	stonith__validate_agent_html(pcmk__output_t *out, va_list args) {
	const char agent = va_arg(args, const char );
	const char device = va_arg(args, const char );
	char output = va_arg(args, char );
	char error_output = va_arg(args, char );
	int rc = va_arg(args, int);

	if (device) {
	char *buf = crm_strdup_printf("Validation of %s on %s %s", agent, device,
	rc ? "failed" : "succeeded");
	pcmk__output_create_html_node(out, "div", NULL, NULL, buf);
	free(buf);
	} else {
	char *buf = crm_strdup_printf("Validation of %s %s", agent,
	rc ? "failed" : "succeeded");
	pcmk__output_create_html_node(out, "div", NULL, NULL, buf);
	free(buf);
	}

	out->subprocess_output(out, rc, output, error_output);
	return rc;
	}

	PCMK__OUTPUT_ARGS("validate", "const char ", "const char ", "char ", "char ", "int")
	int
	stonith__validate_agent_text(pcmk__output_t *out, va_list args) {
	const char agent = va_arg(args, const char );
	const char device = va_arg(args, const char );
	char output = va_arg(args, char );
	char error_output = va_arg(args, char );
	int rc = va_arg(args, int);

	if (device) {
	pcmk__indented_printf(out, "Validation of %s on %s %s\n", agent, device,
	rc ? "failed" : "succeeded");
	} else {
	pcmk__indented_printf(out, "Validation of %s %s\n", agent,
	rc ? "failed" : "succeeded");
	}

	out->subprocess_output(out, rc, output, error_output);
	return rc;
	}

	PCMK__OUTPUT_ARGS("validate", "const char ", "const char ", "char ", "char ", "int")
	int
	stonith__validate_agent_xml(pcmk__output_t *out, va_list args) {
	const char agent = va_arg(args, const char );
	const char device = va_arg(args, const char );
	char output = va_arg(args, char );
	char error_output = va_arg(args, char );
	int rc = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "validate",
	"agent", agent,
	"valid", pcmk__btoa(rc),
	NULL);

	if (device != NULL) {
	crm_xml_add(node, "device", device);
	}

	pcmk__output_xml_push_parent(out, node);
	out->subprocess_output(out, rc, output, error_output);
	pcmk__output_xml_pop_parent(out);

	return rc;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	- { "failed-fencing-history", "default", stonith__failed_history },
	- { "fencing-history", "default", stonith__history },
	- { "full-fencing-history", "default", stonith__full_history },
	- { "full-fencing-history", "xml", stonith__full_history_xml },
	+ { "failed-fencing-list", "default", stonith__failed_history },
	+ { "fencing-list", "default", stonith__history },
	+ { "full-fencing-list", "default", stonith__full_history },
	+ { "full-fencing-list", "xml", stonith__full_history_xml },
	{ "last-fenced", "html", stonith__last_fenced_html },
	{ "last-fenced", "log", stonith__last_fenced_text },
	{ "last-fenced", "text", stonith__last_fenced_text },
	{ "last-fenced", "xml", stonith__last_fenced_xml },
	- { "pending-fencing-actions", "default", stonith__pending_actions },
	+ { "pending-fencing-list", "default", stonith__pending_actions },
	{ "stonith-event", "html", stonith__event_html },
	{ "stonith-event", "log", stonith__event_text },
	{ "stonith-event", "text", stonith__event_text },
	{ "stonith-event", "xml", stonith__event_xml },
	{ "validate", "html", stonith__validate_agent_html },
	{ "validate", "log", stonith__validate_agent_text },
	{ "validate", "text", stonith__validate_agent_text },
	{ "validate", "xml", stonith__validate_agent_xml },

	{ NULL, NULL, NULL }
	};

	void
	stonith__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_mon_curses.c b/tools/crm_mon_curses.c
	index ae22198d97..5c79dd2b77 100644
	--- a/tools/crm_mon_curses.c
	+++ b/tools/crm_mon_curses.c
	@@ -1,427 +1,427 @@
	/*
	* Copyright 2019-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <stdarg.h>
	#include <stdlib.h>
	#include <crm/crm.h>
	#include <crm/common/curses_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/pengine/internal.h>
	#include <glib.h>

	#include "crm_mon.h"

	#if CURSES_ENABLED

	GOptionEntry crm_mon_curses_output_entries[] = {
	{ NULL }
	};

	typedef struct curses_list_data_s {
	unsigned int len;
	char *singular_noun;
	char *plural_noun;
	} curses_list_data_t;

	typedef struct private_data_s {
	GQueue *parent_q;
	} private_data_t;

	static void
	curses_free_priv(pcmk__output_t *out) {
	private_data_t *priv = out->priv;

	if (priv == NULL) {
	return;
	}

	g_queue_free(priv->parent_q);
	free(priv);
	out->priv = NULL;
	}

	static bool
	curses_init(pcmk__output_t *out) {
	private_data_t *priv = NULL;

	/* If curses_init was previously called on this output struct, just return. */
	if (out->priv != NULL) {
	return true;
	} else {
	out->priv = calloc(1, sizeof(private_data_t));
	if (out->priv == NULL) {
	return false;
	}

	priv = out->priv;
	}

	priv->parent_q = g_queue_new();

	initscr();
	cbreak();
	noecho();

	return true;
	}

	static void
	curses_finish(pcmk__output_t out, crm_exit_t exit_status, bool print, void *copy_dest) {
	echo();
	nocbreak();
	endwin();
	}

	static void
	curses_reset(pcmk__output_t *out) {
	CRM_ASSERT(out != NULL);

	curses_free_priv(out);
	curses_init(out);
	}

	static void
	curses_subprocess_output(pcmk__output_t *out, int exit_status,
	const char proc_stdout, const char proc_stderr) {
	if (proc_stdout != NULL) {
	printw("%s\n", proc_stdout);
	}

	if (proc_stderr != NULL) {
	printw("%s\n", proc_stderr);
	}

	clrtoeol();
	refresh();
	}

	/* curses_version is defined in curses.h, so we can't use that name here.
	* Note that this function prints out via text, not with curses.
	*/
	static void
	curses_ver(pcmk__output_t *out, bool extended) {
	if (extended) {
	printf("Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
	} else {
	printf("Pacemaker %s\n", PACEMAKER_VERSION);
	printf("Written by Andrew Beekhof\n");
	}
	}

	G_GNUC_PRINTF(2, 3)
	static void
	curses_error(pcmk__output_t out, const char format, ...) {
	va_list ap;

	/* Informational output does not get indented, to separate it from other
	* potentially indented list output.
	*/
	va_start(ap, format);
	vw_printw(stdscr, format, ap);
	va_end(ap);

	/* Add a newline. */
	addch('\n');

	clrtoeol();
	refresh();
	sleep(2);
	}

	G_GNUC_PRINTF(2, 3)
	static void
	curses_info(pcmk__output_t out, const char format, ...) {
	va_list ap;

	/* Informational output does not get indented, to separate it from other
	* potentially indented list output.
	*/
	va_start(ap, format);
	vw_printw(stdscr, format, ap);
	va_end(ap);

	/* Add a newline. */
	addch('\n');

	clrtoeol();
	refresh();
	}

	static void
	curses_output_xml(pcmk__output_t out, const char name, const char *buf) {
	private_data_t *priv = out->priv;

	CRM_ASSERT(priv != NULL);
	curses_indented_printf(out, "%s", buf);
	}

	G_GNUC_PRINTF(4, 5)
	static void
	curses_begin_list(pcmk__output_t out, const char singular_noun, const char *plural_noun,
	const char *format, ...) {
	private_data_t *priv = out->priv;
	curses_list_data_t *new_list = NULL;
	va_list ap;

	CRM_ASSERT(priv != NULL);

	va_start(ap, format);

	curses_indented_vprintf(out, format, ap);
	printw(":\n");

	va_end(ap);

	new_list = calloc(1, sizeof(curses_list_data_t));
	new_list->len = 0;
	new_list->singular_noun = singular_noun == NULL ? NULL : strdup(singular_noun);
	new_list->plural_noun = plural_noun == NULL ? NULL : strdup(plural_noun);

	g_queue_push_tail(priv->parent_q, new_list);
	}

	G_GNUC_PRINTF(3, 4)
	static void
	curses_list_item(pcmk__output_t out, const char id, const char *format, ...) {
	private_data_t *priv = out->priv;
	va_list ap;

	CRM_ASSERT(priv != NULL);

	va_start(ap, format);

	if (id != NULL) {
	curses_indented_printf(out, "%s: ", id);
	vw_printw(stdscr, format, ap);
	} else {
	curses_indented_vprintf(out, format, ap);
	}

	addch('\n');
	va_end(ap);

	out->increment_list(out);
	}

	static void
	curses_increment_list(pcmk__output_t *out) {
	private_data_t *priv = out->priv;
	gpointer tail;

	CRM_ASSERT(priv != NULL);
	tail = g_queue_peek_tail(priv->parent_q);
	CRM_ASSERT(tail != NULL);
	((curses_list_data_t *) tail)->len++;
	}

	static void
	curses_end_list(pcmk__output_t *out) {
	private_data_t *priv = out->priv;
	curses_list_data_t *node = NULL;

	CRM_ASSERT(priv != NULL);
	node = g_queue_pop_tail(priv->parent_q);

	if (node->singular_noun != NULL && node->plural_noun != NULL) {
	if (node->len == 1) {
	curses_indented_printf(out, "%d %s found\n", node->len, node->singular_noun);
	} else {
	curses_indented_printf(out, "%d %s found\n", node->len, node->plural_noun);
	}
	}

	free(node);
	}

	static bool
	curses_is_quiet(pcmk__output_t *out) {
	return out->quiet;
	}

	static void
	curses_spacer(pcmk__output_t *out) {
	addch('\n');
	}

	pcmk__output_t *
	crm_mon_mk_curses_output(char **argv) {
	pcmk__output_t *retval = calloc(1, sizeof(pcmk__output_t));

	if (retval == NULL) {
	return NULL;
	}

	retval->fmt_name = "console";
	retval->request = argv == NULL ? NULL : g_strjoinv(" ", argv);

	retval->init = curses_init;
	retval->free_priv = curses_free_priv;
	retval->finish = curses_finish;
	retval->reset = curses_reset;

	retval->register_message = pcmk__register_message;
	retval->message = pcmk__call_message;

	retval->subprocess_output = curses_subprocess_output;
	retval->version = curses_ver;
	retval->err = curses_error;
	retval->info = curses_info;
	retval->output_xml = curses_output_xml;

	retval->begin_list = curses_begin_list;
	retval->list_item = curses_list_item;
	retval->increment_list = curses_increment_list;
	retval->end_list = curses_end_list;

	retval->is_quiet = curses_is_quiet;
	retval->spacer = curses_spacer;

	return retval;
	}

	G_GNUC_PRINTF(2, 0)
	void
	curses_indented_vprintf(pcmk__output_t out, const char format, va_list args) {
	int level = 0;
	private_data_t *priv = out->priv;

	CRM_ASSERT(priv != NULL);

	level = g_queue_get_length(priv->parent_q);

	for (int i = 0; i < level; i++) {
	printw(" ");
	}

	if (level > 0) {
	printw("* ");
	}

	vw_printw(stdscr, format, args);

	clrtoeol();
	refresh();
	}

	G_GNUC_PRINTF(2, 3)
	void
	curses_indented_printf(pcmk__output_t out, const char format, ...) {
	va_list ap;

	va_start(ap, format);
	curses_indented_vprintf(out, format, ap);
	va_end(ap);
	}

	PCMK__OUTPUT_ARGS("stonith-event", "stonith_history_t *", "gboolean", "gboolean")
	static int
	stonith_event_console(pcmk__output_t *out, va_list args) {
	stonith_history_t event = va_arg(args, stonith_history_t );
	gboolean full_history = va_arg(args, gboolean);
	gboolean later_succeeded = va_arg(args, gboolean);

	crm_time_t *crm_when = crm_time_new(NULL);
	char *buf = NULL;

	crm_time_set_timet(crm_when, &(event->completed));
	buf = crm_time_as_string(crm_when, crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);

	switch (event->state) {
	case st_failed:
	curses_indented_printf(out, "%s of %s failed: delegate=%s, client=%s, origin=%s, %s='%s'%s\n",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-failed", buf,
	later_succeeded ? " (a later attempt succeeded)" : "");
	break;

	case st_done:
	curses_indented_printf(out, "%s of %s successful: delegate=%s, client=%s, origin=%s, %s='%s'\n",
	stonith_action_str(event->action), event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	full_history ? "completed" : "last-successful", buf);
	break;

	default:
	curses_indented_printf(out, "%s of %s pending: client=%s, origin=%s\n",
	stonith_action_str(event->action), event->target,
	event->client, event->origin);
	break;
	}

	free(buf);
	crm_time_free(crm_when);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "console", pe__ban_text },
	{ "bundle", "console", pe__bundle_text },
	{ "clone", "console", pe__clone_text },
	{ "cluster-counts", "console", pe__cluster_counts_text },
	{ "cluster-dc", "console", pe__cluster_dc_text },
	{ "cluster-options", "console", pe__cluster_options_text },
	{ "cluster-stack", "console", pe__cluster_stack_text },
	{ "cluster-summary", "console", pe__cluster_summary },
	{ "cluster-times", "console", pe__cluster_times_text },
	{ "failed-action", "console", pe__failed_action_text },
	- { "failed-fencing-history", "console", stonith__failed_history },
	- { "fencing-history", "console", stonith__history },
	- { "full-fencing-history", "console", stonith__full_history },
	+ { "failed-fencing-list", "console", stonith__failed_history },
	+ { "fencing-list", "console", stonith__history },
	+ { "full-fencing-list", "console", stonith__full_history },
	{ "group", "console", pe__group_text },
	{ "maint-mode", "console", pe__cluster_maint_mode_text },
	{ "node", "console", pe__node_text },
	{ "node-attribute", "console", pe__node_attribute_text },
	{ "node-list", "console", pe__node_list_text },
	{ "op-history", "console", pe__op_history_text },
	- { "pending-fencing-actions", "console", stonith__pending_actions },
	+ { "pending-fencing-list", "console", stonith__pending_actions },
	{ "primitive", "console", pe__resource_text },
	{ "resource-history", "console", pe__resource_history_text },
	{ "stonith-event", "console", stonith_event_console },
	{ "ticket", "console", pe__ticket_text },

	{ NULL, NULL, NULL }
	};

	void
	crm_mon_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}

	#else

	pcmk__output_t *
	crm_mon_mk_curses_output(char **argv) {
	/* curses was disabled in the build, so fall back to text. */
	return pcmk__mk_text_output(argv);
	}

	G_GNUC_PRINTF(2, 0)
	void
	curses_indented_vprintf(pcmk__output_t out, const char format, va_list args) {
	return;
	}

	G_GNUC_PRINTF(2, 3)
	void
	curses_indented_printf(pcmk__output_t out, const char format, ...) {
	return;
	}

	void
	crm_mon_register_messages(pcmk__output_t *out) {
	return;
	}

	#endif
	diff --git a/tools/crm_mon_print.c b/tools/crm_mon_print.c
	index ce3e47c77a..06840b71e7 100644
	--- a/tools/crm_mon_print.c
	+++ b/tools/crm_mon_print.c
	@@ -1,997 +1,997 @@
	/*
	* Copyright 2019-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <glib.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <time.h>

	#ifndef PCMK__CONFIG_H
	# define PCMK__CONFIG_H
	# include <config.h>
	#endif

	#include <crm/cib/util.h>
	#include <crm/common/curses_internal.h>
	#include <crm/common/iso8601_internal.h>
	#include <crm/common/xml.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/pe_types.h>
	#include <crm/stonith-ng.h>
	#include <crm/common/internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/util.h>
	#include <crm/fencing/internal.h>

	#include "crm_mon.h"

	static int print_rsc_history(pcmk__output_t out, pe_working_set_t data_set,
	pe_node_t node, xmlNode rsc_entry, unsigned int mon_ops,
	GListPtr op_list);
	static int print_node_history(pcmk__output_t out, pe_working_set_t data_set,
	pe_node_t node, xmlNode node_state, gboolean operations,
	unsigned int mon_ops, GListPtr only_node, GListPtr only_rsc);
	static gboolean add_extra_info(pcmk__output_t out, pe_node_t node, GListPtr rsc_list,
	const char attrname, int expected_score);
	static void print_node_attribute(gpointer name, gpointer user_data);
	static int print_node_summary(pcmk__output_t out, pe_working_set_t data_set,
	gboolean operations, unsigned int mon_ops,
	GListPtr only_node, GListPtr only_rsc, gboolean print_spacer);
	static int print_cluster_tickets(pcmk__output_t out, pe_working_set_t data_set,
	gboolean print_spacer);
	static int print_neg_locations(pcmk__output_t out, pe_working_set_t data_set,
	unsigned int mon_ops, const char *prefix,
	GListPtr only_rsc, gboolean print_spacer);
	static int print_node_attributes(pcmk__output_t out, pe_working_set_t data_set,
	unsigned int mon_ops, GListPtr only_node,
	GListPtr only_rsc, gboolean print_spacer);
	static int print_failed_actions(pcmk__output_t out, pe_working_set_t data_set,
	GListPtr only_node, GListPtr only_rsc, gboolean print_spacer);

	static GListPtr
	build_uname_list(pe_working_set_t data_set, const char s) {
	GListPtr unames = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	/* Nothing was given so return a list of all node names. Or, '*' was
	* given. This would normally fall into the pe__unames_with_tag branch
	* where it will return an empty list. Catch it here instead.
	*/
	unames = g_list_prepend(unames, strdup("*"));
	} else {
	pe_node_t *node = pe_find_node(data_set->nodes, s);

	if (node) {
	/* The given string was a valid uname for a node. Return a
	* singleton list containing just that uname.
	*/
	unames = g_list_prepend(unames, strdup(s));
	} else {
	/* The given string was not a valid uname. It's either a tag or
	* it's a typo or something. In the first case, we'll return a
	* list of all the unames of the nodes with the given tag. In the
	* second case, we'll return a NULL pointer and nothing will
	* get displayed.
	*/
	unames = pe__unames_with_tag(data_set, s);
	}
	}

	return unames;
	}

	static GListPtr
	build_rsc_list(pe_working_set_t data_set, const char s) {
	GListPtr resources = NULL;

	if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
	resources = g_list_prepend(resources, strdup("*"));
	} else {
	pe_resource_t *rsc = pe_find_resource_with_flags(data_set->resources, s,
	pe_find_renamed\|pe_find_any);

	if (rsc) {
	/* A colon in the name we were given means we're being asked to filter
	* on a specific instance of a cloned resource. Put that exact string
	* into the filter list. Otherwise, use the printable ID of whatever
	* resource was found that matches what was asked for.
	*/
	if (strstr(s, ":") != NULL) {
	resources = g_list_prepend(resources, strdup(rsc->id));
	} else {
	resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
	}
	} else {
	/* The given string was not a valid resource name. It's either
	* a tag or it's a typo or something. See build_uname_list for
	* more detail.
	*/
	resources = pe__rscs_with_tag(data_set, s);
	}
	}

	return resources;
	}

	static int
	failure_count(pe_working_set_t data_set, pe_node_t node, pe_resource_t rsc, time_t last_failure) {
	return rsc ? pe_get_failcount(node, rsc, last_failure, pe_fc_default,
	NULL, data_set)
	: 0;
	}

	static GListPtr
	get_operation_list(xmlNode *rsc_entry) {
	GListPtr op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {
	const char *task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(rsc_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	int op_rc_i = crm_parse_int(op_rc, "0");

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_casei) \|\| (pcmk__str_eq(task, "probe", pcmk__str_casei) && (op_rc_i == 7))) {
	continue;
	}

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	/*!
	* \internal
	* \brief Print resource operation/failure history
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] node Node that ran this resource.
	* \param[in] rsc_entry Root of XML tree describing resource status.
	* \param[in] mon_ops Bitmask of mon_op_*.
	* \param[in] op_list A list of operations to print.
	*/
	static int
	print_rsc_history(pcmk__output_t out, pe_working_set_t data_set, pe_node_t *node,
	xmlNode *rsc_entry, unsigned int mon_ops, GListPtr op_list)
	{
	GListPtr gIter = NULL;
	int rc = pcmk_rc_no_output;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(xml_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	int op_rc_i = crm_parse_int(op_rc, "0");

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, CRMD_ACTION_STATUS, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = failure_count(data_set, node, rsc, &last_failure);

	out->message(out, "resource-history", rsc, rsc_id, TRUE, failcount, last_failure, TRUE);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, pcmk_is_set(mon_ops, mon_op_print_timing));
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \brief Print node operation/failure history
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] node_state Root of XML tree describing node status.
	* \param[in] operations Whether to print operations or just failcounts.
	* \param[in] mon_ops Bitmask of mon_op_*.
	*/
	static int
	print_node_history(pcmk__output_t out, pe_working_set_t data_set,
	pe_node_t node, xmlNode node_state, gboolean operations,
	unsigned int mon_ops, GListPtr only_node, GListPtr only_rsc)
	{
	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = pcmk__xe_first_child(lrm_rsc); rsc_entry != NULL;
	rsc_entry = pcmk__xe_next(rsc_entry)) {

	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {
	continue;
	}

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (uber_parent(rsc)->variant == pe_group) {
	if (!pcmk__str_in_list(only_rsc, rsc_printable_id(rsc)) &&
	!pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(rsc)))) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (operations == FALSE) {
	time_t last_failure = 0;
	int failcount = failure_count(data_set, node, rsc, &last_failure);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, get_resource_display_options(mon_ops),
	FALSE, NULL,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	only_node, only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, FALSE,
	failcount, last_failure, FALSE);
	} else {
	GListPtr op_list = get_operation_list(rsc_entry);

	if (op_list == NULL) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, get_resource_display_options(mon_ops),
	FALSE, NULL,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	only_node, only_rsc);
	}

	print_rsc_history(out, data_set, node, rsc_entry, mon_ops, op_list);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] out The output functions structure.
	* \param[in] node Node that ran this resource.
	* \param[in] rsc_list The list of resources for this node.
	* \param[in] attrname The attribute to find.
	* \param[out] expected_score The expected value for this attribute.
	*
	* \return TRUE if extended information should be printed, FALSE otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static gboolean
	add_extra_info(pcmk__output_t out, pe_node_t node, GListPtr rsc_list,
	const char attrname, int expected_score)
	{
	GListPtr gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, "type");
	const char *name = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(out, node, rsc->children, attrname, expected_score)) {
	return TRUE;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	return FALSE;
	}

	name = g_hash_table_lookup(rsc->parameters, "name");

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	/* int value = crm_parse_int(attrvalue, "0"); */
	const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
	const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	/* pingd multiplier is the same as the default value. */
	expected_score = host_list_num crm_parse_int(multiplier, "1");

	return TRUE;
	}
	}
	return FALSE;
	}

	/* structure for passing multiple user data to g_list_foreach() */
	struct mon_attr_data {
	pcmk__output_t *out;
	pe_node_t *node;
	};

	static void
	print_node_attribute(gpointer name, gpointer user_data)
	{
	const char *value = NULL;
	int expected_score = 0;
	gboolean add_extra = FALSE;
	struct mon_attr_data data = (struct mon_attr_data ) user_data;

	value = pe_node_attribute_raw(data->node, name);

	add_extra = add_extra_info(data->out, data->node, data->node->details->running_rsc,
	name, &expected_score);

	/* Print attribute name and value */
	data->out->message(data->out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	/*!
	* \internal
	* \brief Print history for all nodes.
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] operations Whether to print operations or just failcounts.
	* \param[in] mon_ops Bitmask of mon_op_*.
	*/
	static int
	print_node_summary(pcmk__output_t out, pe_working_set_t data_set,
	gboolean operations, unsigned int mon_ops, GListPtr only_node,
	GListPtr only_rsc, gboolean print_spacer)
	{
	xmlNode *node_state = NULL;
	xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	/* Print each node in the CIB status */
	for (node_state = pcmk__xe_first_child(cib_status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {
	pe_node_t *node;

	if (!pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	continue;
	}

	node = pe_find_node_id(data_set->nodes, ID(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(only_node, node->details->uname)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, operations ? "Operations" : "Migration Summary");

	print_node_history(out, data_set, node, node_state, operations, mon_ops,
	only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \brief Print all tickets.
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	*/
	static int
	print_cluster_tickets(pcmk__output_t out, pe_working_set_t data_set,
	gboolean print_spacer)
	{
	GHashTableIter iter;
	gpointer key, value;

	if (g_hash_table_size(data_set->tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, data_set->tickets);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	pe_ticket_t ticket = (pe_ticket_t ) value;
	out->message(out, "ticket", ticket);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Print section for negative location constraints
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] mon_ops Bitmask of mon_op_*.
	* \param[in] prefix ID prefix to filter results by.
	*/
	static int
	print_neg_locations(pcmk__output_t out, pe_working_set_t data_set,
	unsigned int mon_ops, const char *prefix, GListPtr only_rsc,
	gboolean print_spacer)
	{
	GListPtr gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
	pe__location_t *location = gIter->data;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix))
	continue;

	if (!pcmk__str_in_list(only_rsc, rsc_printable_id(location->rsc_lh)) &&
	!pcmk__str_in_list(only_rsc, rsc_printable_id(uber_parent(location->rsc_lh)))) {
	continue;
	}

	for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_node_t node = (pe_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail));
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \brief Print node attributes section
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] mon_ops Bitmask of mon_op_*.
	*/
	static int
	print_node_attributes(pcmk__output_t out, pe_working_set_t data_set,
	unsigned int mon_ops, GListPtr only_node,
	GListPtr only_rsc, gboolean print_spacer)
	{
	GListPtr gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Unpack all resource parameters (it would be more efficient to do this
	* only when needed for the first time in add_extra_info())
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	crm_mon_get_parameters(gIter->data, data_set);
	}

	/* Display each node's attributes */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	struct mon_attr_data data;

	data.out = out;
	data.node = (pe_node_t *) gIter->data;

	if (data.node && data.node->details && data.node->details->online) {
	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key, value;

	g_hash_table_iter_init(&iter, data.node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, &value)) {
	attr_list = append_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(only_node, data.node->details->uname)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", data.node, get_resource_display_options(mon_ops),
	FALSE, NULL,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	only_node, only_rsc);
	g_list_foreach(attr_list, print_node_attribute, &data);
	g_list_free(attr_list);
	out->end_list(out);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \brief Print a section for failed actions
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	*/
	static int
	print_failed_actions(pcmk__output_t out, pe_working_set_t data_set,
	GListPtr only_node, GListPtr only_rsc, gboolean print_spacer)
	{
	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	const char *id = NULL;

	if (xmlChildElementCount(data_set->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xml_first_child(data_set->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	char *rsc = NULL;

	if (!pcmk__str_in_list(only_node, crm_element_value(xml_op, XML_ATTR_UNAME))) {
	continue;
	}

	id = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if (parse_op_key(id ? id : ID(xml_op), &rsc, NULL, NULL) == FALSE) {
	continue;
	}

	if (!pcmk__str_in_list(only_rsc, rsc)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	#define CHECK_RC(retcode, retval) \
	if (retval == pcmk_rc_ok) { \
	retcode = pcmk_rc_ok; \
	}

	/*!
	* \internal
	* \brief Top-level printing function for text/curses output.
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] stonith_history List of stonith actions.
	* \param[in] mon_ops Bitmask of mon_op_*.
	* \param[in] show Bitmask of mon_show_*.
	* \param[in] prefix ID prefix to filter results by.
	*/
	void
	print_status(pcmk__output_t out, pe_working_set_t data_set,
	stonith_history_t *stonith_history, unsigned int mon_ops,
	unsigned int show, char prefix, char only_node, char *only_rsc)
	{
	GListPtr unames = NULL;
	GListPtr resources = NULL;

	unsigned int print_opts = get_resource_display_options(mon_ops);
	int rc = pcmk_rc_no_output;

	CHECK_RC(rc, out->message(out, "cluster-summary", data_set,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(show, mon_show_stack),
	pcmk_is_set(show, mon_show_dc),
	pcmk_is_set(show, mon_show_times),
	pcmk_is_set(show, mon_show_counts),
	pcmk_is_set(show, mon_show_options)));

	unames = build_uname_list(data_set, only_node);
	resources = build_rsc_list(data_set, only_rsc);

	if (pcmk_is_set(show, mon_show_nodes) && unames) {
	PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
	CHECK_RC(rc, out->message(out, "node-list", data_set->nodes, unames,
	resources, print_opts,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node)));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(show, mon_show_resources)) {
	CHECK_RC(rc, out->message(out, "resource-list", data_set, print_opts,
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	pcmk_is_set(mon_ops, mon_op_inactive_resources),
	pcmk_is_set(mon_ops, mon_op_print_brief), TRUE, unames,
	resources, rc == pcmk_rc_ok));
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(show, mon_show_attributes)) {
	CHECK_RC(rc, print_node_attributes(out, data_set, mon_ops, unames, resources,
	rc == pcmk_rc_ok));
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_is_set(show, mon_show_operations)
	\|\| pcmk_is_set(show, mon_show_failcounts)) {

	CHECK_RC(rc, print_node_summary(out, data_set,
	pcmk_is_set(show, mon_show_operations),
	mon_ops, unames, resources,
	(rc == pcmk_rc_ok)));
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(show, mon_show_failures)
	&& xml_has_children(data_set->failed)) {

	CHECK_RC(rc, print_failed_actions(out, data_set, unames, resources,
	rc == pcmk_rc_ok));
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(show, mon_show_fence_failed)
	&& pcmk_is_set(mon_ops, mon_op_fence_history)) {

	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));

	if (hp) {
	- CHECK_RC(rc, out->message(out, "failed-fencing-history", stonith_history, unames,
	+ CHECK_RC(rc, out->message(out, "failed-fencing-list", stonith_history, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history),
	rc == pcmk_rc_ok));
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(show, mon_show_tickets)) {
	CHECK_RC(rc, print_cluster_tickets(out, data_set, rc == pcmk_rc_ok));
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(show, mon_show_bans)) {
	CHECK_RC(rc, print_neg_locations(out, data_set, mon_ops, prefix, resources,
	rc == pcmk_rc_ok));
	}

	/* Print stonith history */
	if (pcmk_is_set(mon_ops, mon_op_fence_history)) {
	if (pcmk_is_set(show, mon_show_fence_worked)) {
	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));

	if (hp) {
	- CHECK_RC(rc, out->message(out, "fencing-history", hp, unames,
	+ CHECK_RC(rc, out->message(out, "fencing-list", hp, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history),
	rc == pcmk_rc_ok));
	}
	} else if (pcmk_is_set(show, mon_show_fence_pending)) {
	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_pending, NULL);

	if (hp) {
	- CHECK_RC(rc, out->message(out, "pending-fencing-actions", hp, unames,
	+ CHECK_RC(rc, out->message(out, "pending-fencing-list", hp, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history),
	rc == pcmk_rc_ok));
	}
	}
	}

	g_list_free_full(unames, free);
	}

	/*!
	* \internal
	* \brief Top-level printing function for XML output.
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] stonith_history List of stonith actions.
	* \param[in] mon_ops Bitmask of mon_op_*.
	* \param[in] show Bitmask of mon_show_*.
	* \param[in] prefix ID prefix to filter results by.
	*/
	void
	print_xml_status(pcmk__output_t out, pe_working_set_t data_set,
	crm_exit_t history_rc, stonith_history_t *stonith_history,
	unsigned int mon_ops, unsigned int show, char *prefix,
	char only_node, char only_rsc)
	{
	GListPtr unames = NULL;
	GListPtr resources = NULL;
	unsigned int print_opts = get_resource_display_options(mon_ops);

	out->message(out, "cluster-summary", data_set,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(show, mon_show_stack),
	pcmk_is_set(show, mon_show_dc),
	pcmk_is_set(show, mon_show_times),
	pcmk_is_set(show, mon_show_counts),
	pcmk_is_set(show, mon_show_options));

	unames = build_uname_list(data_set, only_node);
	resources = build_rsc_list(data_set, only_rsc);

	/* NODES */
	if (pcmk_is_set(show, mon_show_nodes)) {
	out->message(out, "node-list", data_set->nodes, unames,
	resources, print_opts,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(show, mon_show_resources)) {
	out->message(out, "resource-list", data_set, print_opts,
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	pcmk_is_set(mon_ops, mon_op_inactive_resources),
	FALSE, FALSE, unames, resources, FALSE);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(show, mon_show_attributes)) {
	print_node_attributes(out, data_set, mon_ops, unames, resources, FALSE);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_is_set(show, mon_show_operations)
	\|\| pcmk_is_set(show, mon_show_failcounts)) {

	print_node_summary(out, data_set,
	pcmk_is_set(show, mon_show_operations),
	mon_ops, unames, resources, FALSE);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(show, mon_show_failures)
	&& xml_has_children(data_set->failed)) {

	print_failed_actions(out, data_set, unames, resources, FALSE);
	}

	/* Print stonith history */
	if (pcmk_is_set(show, mon_show_fencing_all)
	&& pcmk_is_set(mon_ops, mon_op_fence_history)) {

	- out->message(out, "full-fencing-history", history_rc, stonith_history,
	+ out->message(out, "full-fencing-list", history_rc, stonith_history,
	unames, pcmk_is_set(mon_ops, mon_op_fence_full_history),
	FALSE);
	}

	/* Print tickets if requested */
	if (pcmk_is_set(show, mon_show_tickets)) {
	print_cluster_tickets(out, data_set, FALSE);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(show, mon_show_bans)) {
	print_neg_locations(out, data_set, mon_ops, prefix, resources, FALSE);
	}

	g_list_free_full(unames, free);
	g_list_free_full(resources, free);
	}

	/*!
	* \internal
	* \brief Top-level printing function for HTML output.
	*
	* \param[in] out The output functions structure.
	* \param[in] data_set Cluster state to display.
	* \param[in] stonith_history List of stonith actions.
	* \param[in] mon_ops Bitmask of mon_op_*.
	* \param[in] show Bitmask of mon_show_*.
	* \param[in] prefix ID prefix to filter results by.
	*/
	int
	print_html_status(pcmk__output_t out, pe_working_set_t data_set,
	stonith_history_t *stonith_history, unsigned int mon_ops,
	unsigned int show, char prefix, char only_node,
	char *only_rsc)
	{
	GListPtr unames = NULL;
	GListPtr resources = NULL;

	unsigned int print_opts = get_resource_display_options(mon_ops);

	out->message(out, "cluster-summary", data_set,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(show, mon_show_stack),
	pcmk_is_set(show, mon_show_dc),
	pcmk_is_set(show, mon_show_times),
	pcmk_is_set(show, mon_show_counts),
	pcmk_is_set(show, mon_show_options));

	unames = build_uname_list(data_set, only_node);
	resources = build_rsc_list(data_set, only_rsc);

	/* NODE LIST */
	if (pcmk_is_set(show, mon_show_nodes) && unames) {
	out->message(out, "node-list", data_set->nodes, unames,
	resources, print_opts,
	pcmk_is_set(mon_ops, mon_op_print_clone_detail),
	pcmk_is_set(mon_ops, mon_op_print_brief),
	pcmk_is_set(mon_ops, mon_op_group_by_node));
	}

	/* Print resources section, if needed */
	if (pcmk_is_set(show, mon_show_resources)) {
	out->message(out, "resource-list", data_set, print_opts,
	pcmk_is_set(mon_ops, mon_op_group_by_node),
	pcmk_is_set(mon_ops, mon_op_inactive_resources),
	pcmk_is_set(mon_ops, mon_op_print_brief), TRUE, unames,
	resources, FALSE);
	}

	/* print Node Attributes section if requested */
	if (pcmk_is_set(show, mon_show_attributes)) {
	print_node_attributes(out, data_set, mon_ops, unames, resources, FALSE);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (pcmk_is_set(show, mon_show_operations)
	\|\| pcmk_is_set(show, mon_show_failcounts)) {

	print_node_summary(out, data_set,
	pcmk_is_set(show, mon_show_operations),
	mon_ops, unames, resources, FALSE);
	}

	/* If there were any failed actions, print them */
	if (pcmk_is_set(show, mon_show_failures)
	&& xml_has_children(data_set->failed)) {

	print_failed_actions(out, data_set, unames, resources, FALSE);
	}

	/* Print failed stonith actions */
	if (pcmk_is_set(show, mon_show_fence_failed)
	&& pcmk_is_set(mon_ops, mon_op_fence_history)) {

	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_eq,
	GINT_TO_POINTER(st_failed));

	if (hp) {
	- out->message(out, "failed-fencing-history", stonith_history, unames,
	+ out->message(out, "failed-fencing-list", stonith_history, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history), FALSE);
	}
	}

	/* Print stonith history */
	if (pcmk_is_set(mon_ops, mon_op_fence_history)) {
	if (pcmk_is_set(show, mon_show_fence_worked)) {
	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_neq,
	GINT_TO_POINTER(st_failed));

	if (hp) {
	- out->message(out, "fencing-history", hp, unames,
	+ out->message(out, "fencing-list", hp, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history),
	FALSE);
	}
	} else if (pcmk_is_set(show, mon_show_fence_pending)) {
	stonith_history_t *hp = stonith__first_matching_event(stonith_history, stonith__event_state_pending, NULL);

	if (hp) {
	- out->message(out, "pending-fencing-actions", hp, unames,
	+ out->message(out, "pending-fencing-list", hp, unames,
	pcmk_is_set(mon_ops, mon_op_fence_full_history),
	FALSE);
	}
	}
	}

	/* Print tickets if requested */
	if (pcmk_is_set(show, mon_show_tickets)) {
	print_cluster_tickets(out, data_set, FALSE);
	}

	/* Print negative location constraints if requested */
	if (pcmk_is_set(show, mon_show_bans)) {
	print_neg_locations(out, data_set, mon_ops, prefix, resources, FALSE);
	}

	g_list_free_full(unames, free);
	g_list_free_full(resources, free);
	return 0;
	}
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index 2c62ff6d1c..95c72fcaab 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,2037 +1,2037 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_resource.h>
	#include <crm/lrmd_internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/lists_internal.h>
	#include <pacemaker-internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/cib/internal.h>

	#define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"

	enum rsc_command {
	cmd_none = 0, // No command option given (yet)
	cmd_ban,
	cmd_cleanup,
	cmd_clear,
	cmd_colocations,
	cmd_colocations_deep,
	cmd_cts,
	cmd_delete,
	cmd_delete_param,
	cmd_execute_agent,
	cmd_fail,
	cmd_get_param,
	cmd_get_property,
	cmd_list_active_ops,
	cmd_list_agents,
	cmd_list_all_ops,
	cmd_list_alternatives,
	cmd_list_instances,
	cmd_list_providers,
	cmd_list_resources,
	cmd_list_standards,
	cmd_locate,
	cmd_metadata,
	cmd_move,
	cmd_query_raw_xml,
	cmd_query_xml,
	cmd_refresh,
	cmd_restart,
	cmd_set_param,
	cmd_set_property,
	cmd_wait,
	cmd_why,
	};

	struct {
	enum rsc_command rsc_cmd; // The crm_resource command to perform
	const char *attr_set_type;
	int cib_options;
	gboolean clear_expired;
	int find_flags; /* Flags to use when searching for resource */
	gboolean force;
	gchar *host_uname;
	gchar *interval_spec;
	gchar *move_lifetime;
	gchar *operation;
	GHashTable *override_params;
	gchar *prop_id;
	char *prop_name;
	gchar *prop_set;
	gchar *prop_value;
	gboolean recursive;
	gchar **remainder;
	gboolean require_cib; // Whether command requires CIB connection
	gboolean require_crmd; /* whether command requires controller connection */
	gboolean require_dataset; /* whether command requires populated dataset instance */
	gboolean require_resource; /* whether command requires that resource be specified */
	int resource_verbose;
	gchar *rsc_id;
	gchar *rsc_type;
	gboolean promoted_role_only;
	int timeout_ms;
	char *agent_spec; // Standard and/or provider and/or agent
	char *v_agent;
	char *v_class;
	char *v_provider;
	gboolean validate_cmdline; /* whether we are just validating based on command line options */
	GHashTable *validate_options;
	gchar *xml_file;
	} options = {
	.attr_set_type = XML_TAG_ATTR_SETS,
	.cib_options = cib_sync_call,
	.require_cib = TRUE,
	.require_dataset = TRUE,
	.require_resource = TRUE,
	};

	#if 0
	// @COMPAT @TODO enable this at next backward compatibility break
	#define SET_COMMAND(cmd) do { \
	if (options.rsc_cmd != cmd_none) { \
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE, \
	"Only one command option may be specified"); \
	return FALSE; \
	} \
	options.rsc_cmd = (cmd); \
	} while (0)
	#else
	#define SET_COMMAND(cmd) do { options.rsc_cmd = (cmd); } while (0)
	#endif

	gboolean agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_agents_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_providers_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_standards_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean metadata_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean option_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean restart_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static crm_exit_t exit_code = CRM_EX_OK;
	static pcmk__output_t *out = NULL;

	// Things that should be cleaned up on exit
	static GError *error = NULL;
	static GMainLoop *mainloop = NULL;
	static cib_t *cib_conn = NULL;
	static pcmk_ipc_api_t *controld_api = NULL;
	static pe_working_set_t *data_set = NULL;

	#define MESSAGE_TIMEOUT_S 60

	#define INDENT " "

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	// Clean up and exit
	static crm_exit_t
	bye(crm_exit_t ec)
	{
	if (error != NULL) {
	if (out != NULL) {
	out->err(out, "%s: %s", g_get_prgname(), error->message);
	} else {
	fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
	}

	g_clear_error(&error);
	}

	if (out != NULL) {
	out->finish(out, ec, true, NULL);
	pcmk__output_free(out);
	}

	if (cib_conn != NULL) {
	cib_t *save_cib_conn = cib_conn;

	cib_conn = NULL; // Ensure we can't free this twice
	save_cib_conn->cmds->signoff(save_cib_conn);
	cib_delete(save_cib_conn);
	}
	if (controld_api != NULL) {
	pcmk_ipc_api_t *save_controld_api = controld_api;

	controld_api = NULL; // Ensure we can't free this twice
	pcmk_free_ipc_api(save_controld_api);
	}
	if (mainloop != NULL) {
	g_main_loop_unref(mainloop);
	mainloop = NULL;
	}
	pe_free_working_set(data_set);
	data_set = NULL;
	crm_exit(ec);
	return ec;
	}

	static void
	quit_main_loop(crm_exit_t ec)
	{
	exit_code = ec;
	if (mainloop != NULL) {
	GMainLoop *mloop = mainloop;

	mainloop = NULL; // Don't re-enter this block
	pcmk_quit_main_loop(mloop, 10);
	g_main_loop_unref(mloop);
	}
	}

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	// Start with newline because "Waiting for ..." message doesn't have one
	if (error != NULL) {
	g_clear_error(&error);
	}

	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
	"Aborting because no messages received in %d seconds", MESSAGE_TIMEOUT_S);

	quit_main_loop(CRM_EX_TIMEOUT);
	return FALSE;
	}

	static void
	controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void event_data, void user_data)
	{
	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
	crm_info("Connection to controller was terminated");
	}
	quit_main_loop(exit_code);
	break;

	case pcmk_ipc_event_reply:
	if (status != CRM_EX_OK) {
	out->err(out, "Error: bad reply from controller: %s",
	crm_exit_str(status));
	pcmk_disconnect_ipc(api);
	quit_main_loop(status);
	} else {
	if ((pcmk_controld_api_replies_expected(api) == 0)
	&& mainloop && g_main_loop_is_running(mainloop)) {
	out->info(out, "... got reply (done)");
	crm_debug("Got all the replies we expected");
	pcmk_disconnect_ipc(api);
	quit_main_loop(CRM_EX_OK);
	} else {
	out->info(out, "... got reply");
	}
	}
	break;

	default:
	break;
	}
	}

	static void
	start_mainloop(pcmk_ipc_api_t *capi)
	{
	unsigned int count = pcmk_controld_api_replies_expected(capi);

	if (count > 0) {
	out->info(out, "Waiting for %d %s from the controller",
	count, pcmk__plural_alt(count, "reply", "replies"));
	exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
	mainloop = g_main_loop_new(NULL, FALSE);
	g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
	g_main_loop_run(mainloop);
	}
	}

	static int
	compare_id(gconstpointer a, gconstpointer b)
	{
	return strcmp((const char )a, (const char )b);
	}

	static GListPtr
	build_constraint_list(xmlNode *root)
	{
	GListPtr retval = NULL;
	xmlNode *cib_constraints = NULL;
	xmlXPathObjectPtr xpathObj = NULL;
	int ndx = 0;

	cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, root);
	xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);

	for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
	xmlNode *match = getXpathResult(xpathObj, ndx);
	retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
	}

	freeXpathObject(xpathObj);
	return retval;
	}

	/* short option letters still available: eEJkKXyYZ */

	static GOptionEntry query_entries[] = {
	{ "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all cluster resources with status",
	NULL },
	{ "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List IDs of all instantiated resources (individual members\n"
	INDENT "rather than groups etc.)",
	NULL },
	{ "list-cts", 'c', G_OPTION_FLAG_HIDDEN\|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	NULL,
	NULL },
	{ "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List active resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_standards_cb,
	"List supported standards",
	NULL },
	{ "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_providers_cb,
	"List all available OCF providers",
	NULL },
	{ "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_agents_cb,
	"List all agents available for the named standard and/or provider",
	"STD/PROV" },
	{ "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_alternatives_cb,
	"List all available providers for the named OCF agent",
	"AGENT" },
	{ "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	metadata_cb,
	"Show the metadata for the named class:provider:agent",
	"SPEC" },
	{ "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (after any template expansion)",
	NULL },
	{ "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (before any template expansion)",
	NULL },
	{ "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named parameter for resource (use instance attribute\n"
	INDENT "unless --meta or --utilization is specified)",
	"PARAM" },
	{ "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named property of resource ('class', 'type', or 'provider') "
	"(requires --resource)",
	"PROPERTY" },
	{ "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show node(s) currently running resource",
	NULL },
	{ "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Display the (co)location constraints that apply to a resource\n"
	INDENT "and the resources is it colocated with",
	NULL },
	{ "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Display the (co)location constraints that apply to a resource",
	NULL },
	{ "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
	"Show why resources are not running, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },

	{ NULL }
	};

	static GOptionEntry command_entries[] = {
	{ "validate", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"Validate resource configuration by calling agent's validate-all\n"
	INDENT "action. The configuration may be specified either by giving an\n"
	INDENT "existing resource name with -r, or by specifying --class,\n"
	INDENT "--agent, and --provider arguments, along with any number of\n"
	INDENT "--option arguments.",
	NULL },
	{ "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"If resource has any past failures, clear its history and fail\n"
	INDENT "count. Optionally filtered by --resource, --node, --operation\n"
	INDENT "and --interval (otherwise all). --operation and --interval\n"
	INDENT "apply to fail counts, but entire history is always clear, to\n"
	INDENT "allow current state to be rechecked. If the named resource is\n"
	INDENT "part of a group, or one numbered instance of a clone or bundled\n"
	INDENT "resource, the clean-up applies to the whole collective resource\n"
	INDENT "unless --force is given.",
	NULL },
	{ "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"Delete resource's history (including failures) so its current state\n"
	INDENT "is rechecked. Optionally filtered by --resource and --node\n"
	INDENT "(otherwise all). If the named resource is part of a group, or one\n"
	INDENT "numbered instance of a clone or bundled resource, the refresh\n"
	INDENT "applies to the whole collective resource unless --force is given.",
	NULL },
	{ "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Set named parameter for resource (requires -v). Use instance\n"
	INDENT "attribute unless --meta or --utilization is specified.",
	"PARAM" },
	{ "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Delete named parameter for resource. Use instance attribute\n"
	INDENT "unless --meta or --utilization is specified.",
	"PARAM" },
	{ "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
	"Set named property of resource ('class', 'type', or 'provider') "
	"(requires -r, -t, -v)",
	"PROPERTY" },

	{ NULL }
	};

	static GOptionEntry location_entries[] = {
	{ "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to move resource. If --node is specified,\n"
	INDENT "the constraint will be to move to that node, otherwise it\n"
	INDENT "will be to ban the current node. Unless --force is specified\n"
	INDENT "this will return an error if the resource is already running\n"
	INDENT "on the specified node. If --force is specified, this will\n"
	INDENT "always ban the current node.\n"
	INDENT "Optional: --lifetime, --master. NOTE: This may prevent the\n"
	INDENT "resource from running on its previous location until the\n"
	INDENT "implicit constraint expires or is removed with --clear.",
	NULL },
	{ "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to keep resource off a node.\n"
	INDENT "Optional: --node, --lifetime, --master.\n"
	INDENT "NOTE: This will prevent the resource from running on the\n"
	INDENT "affected node until the implicit constraint expires or is\n"
	INDENT "removed with --clear. If --node is not specified, it defaults\n"
	INDENT "to the node currently running the resource for primitives\n"
	INDENT "and groups, or the master for promotable clones with\n"
	INDENT "promoted-max=1 (all other situations result in an error as\n"
	INDENT "there is no sane default).",
	NULL },
	{ "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Remove all constraints created by the --ban and/or --move\n"
	INDENT "commands. Requires: --resource. Optional: --node, --master,\n"
	INDENT "--expired. If --node is not specified, all constraints created\n"
	INDENT "by --ban and --move will be removed for the named resource. If\n"
	INDENT "--node and --force are specified, any constraint created by\n"
	INDENT "--move will be cleared, even if it is not for the specified\n"
	INDENT "node. If --expired is specified, only those constraints whose\n"
	INDENT "lifetimes have expired will be removed.",
	NULL },
	{ "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
	"Modifies the --clear argument to remove constraints with\n"
	INDENT "expired lifetimes.",
	NULL },
	{ "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
	"Lifespan (as ISO 8601 duration) of created constraints (with\n"
	INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
	"TIMESPEC" },
	{ "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.promoted_role_only,
	"Limit scope of command to Master role (with -B, -M, -U). For\n"
	INDENT "-B and -M the previous master may remain active in the Slave role.",
	NULL },

	{ NULL }
	};

	static GOptionEntry advanced_entries[] = {
	{ "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
	"(Advanced) Delete a resource from the CIB. Required: -t",
	NULL },
	{ "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
	"(Advanced) Tell the cluster this resource has failed",
	NULL },
	{ "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
	"(Advanced) Tell the cluster to restart this resource and\n"
	INDENT "anything that depends on it",
	NULL },
	{ "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
	"(Advanced) Wait until the cluster settles into a stable state",
	NULL },
	{ "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and demote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and stop a resource on the local node",
	NULL },
	{ "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and start a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and promote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-check", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and check the state of a resource on\n"
	INDENT "the local node",
	NULL },

	{ NULL }
	};

	static GOptionEntry validate_entries[] = {
	{ "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
	"The standard the resource agent confirms to (for example, ocf).\n"
	INDENT "Use with --agent, --provider, --option, and --validate.",
	"CLASS" },
	{ "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The agent to use (for example, IPaddr). Use with --class,\n"
	INDENT "--provider, --option, and --validate.",
	"AGENT" },
	{ "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The vendor that supplies the resource agent (for example,\n"
	INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
	"PROVIDER" },
	{ "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
	"Specify a device configuration parameter as NAME=VALUE (may be\n"
	INDENT "specified multiple times). Use with --validate and without the\n"
	INDENT "-r option.",
	"PARAM" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
	"Node name",
	"NAME" },
	{ "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
	"Follow colocation chains when using --set-parameter",
	NULL },
	{ "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
	"Resource XML element (primitive, group, etc.) (with -D)",
	"ELEMENT" },
	{ "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
	"Value to use with -p",
	"PARAM" },
	{ "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource meta-attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource utilization attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
	"Operation to clear instead of all (with -C -r)",
	"OPERATION" },
	{ "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
	"Interval of operation to clear (default 0) (with -C -r -n)",
	"N" },
	{ "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
	"(Advanced) XML ID of attributes element to use (with -p, -d)",
	"ID" },
	{ "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
	"(Advanced) XML ID of nvpair element to use (with -p, -d)",
	"ID" },
	{ "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
	"(Advanced) Abort if command does not finish in this time (with\n"
	INDENT "--restart, --wait, --force-*)",
	"N" },
	{ "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
	"If making CIB changes, do so regardless of quorum. See help for\n"
	INDENT "individual commands for additional behavior.",
	NULL },
	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
	NULL,
	"FILE" },
	{ "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
	NULL,
	"HOST" },

	{ NULL }
	};

	gboolean
	agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.validate_cmdline = TRUE;
	options.require_resource = FALSE;

	if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
	if (options.v_provider) {
	free(options.v_provider);
	}
	options.v_provider = strdup(optarg);
	} else {
	if (options.v_agent) {
	free(options.v_agent);
	}
	options.v_agent = strdup(optarg);
	}

	return TRUE;
	}

	gboolean
	attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
	options.attr_set_type = XML_TAG_META_SETS;
	} else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
	options.attr_set_type = XML_TAG_UTILIZATION;
	}

	return TRUE;
	}

	gboolean
	class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (!(pcmk_get_ra_caps(optarg) & pcmk_ra_cap_params)) {
	if (!out->is_quiet(out)) {
	g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
	"Standard %s does not support parameters\n", optarg);
	}
	return FALSE;

	} else {
	if (options.v_class != NULL) {
	free(options.v_class);
	}

	options.v_class = strdup(optarg);
	}

	options.validate_cmdline = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
	SET_COMMAND(cmd_cleanup);
	} else {
	SET_COMMAND(cmd_refresh);
	}

	options.require_resource = FALSE;
	if (getenv("CIB_file") == NULL) {
	options.require_crmd = TRUE;
	}
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	gboolean
	delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_dataset = FALSE;
	SET_COMMAND(cmd_delete);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.clear_expired = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	static void
	get_agent_spec(const gchar *optarg)
	{
	options.require_cib = FALSE;
	options.require_dataset = FALSE;
	options.require_resource = FALSE;
	if (options.agent_spec != NULL) {
	free(options.agent_spec);
	options.agent_spec = NULL;
	}
	if (optarg != NULL) {
	options.agent_spec = strdup(optarg);
	}
	}

	gboolean
	list_agents_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_agents);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_providers_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_providers);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_standards_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_standards);
	options.require_cib = FALSE;
	options.require_dataset = FALSE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_list_alternatives);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	metadata_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_metadata);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	option_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	char *name = NULL;
	char *value = NULL;

	if (pcmk_scan_nvpair(optarg, &name, &value) != 2) {
	return FALSE;
	}
	if (options.validate_options == NULL) {
	options.validate_options = crm_str_table_new();
	}
	g_hash_table_replace(options.validate_options, name, value);
	return TRUE;
	}

	gboolean
	fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_crmd = TRUE;
	SET_COMMAND(cmd_fail);
	return TRUE;
	}

	gboolean
	flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_clear);
	} else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_ban);
	} else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_move);
	} else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_any;
	SET_COMMAND(cmd_query_xml);
	} else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_any;
	SET_COMMAND(cmd_query_raw_xml);
	} else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_locate);
	} else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_colocations_deep);
	} else {
	options.find_flags = pe_find_renamed\|pe_find_anon;
	SET_COMMAND(cmd_colocations);
	}

	return TRUE;
	}

	gboolean
	get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
	SET_COMMAND(cmd_get_param);
	} else {
	SET_COMMAND(cmd_get_property);
	}

	if (options.prop_name) {
	free(options.prop_name);
	}

	options.prop_name = strdup(optarg);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
	SET_COMMAND(cmd_cts);
	} else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
	SET_COMMAND(cmd_list_resources);
	} else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
	SET_COMMAND(cmd_list_instances);
	} else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
	SET_COMMAND(cmd_list_active_ops);
	} else {
	SET_COMMAND(cmd_list_all_ops);
	}

	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
	SET_COMMAND(cmd_set_param);
	} else {
	SET_COMMAND(cmd_delete_param);
	}

	if (options.prop_name) {
	free(options.prop_name);
	}

	options.prop_name = strdup(optarg);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_dataset = FALSE;

	if (options.prop_name) {
	free(options.prop_name);
	}

	options.prop_name = strdup(optarg);
	SET_COMMAND(cmd_set_property);
	options.find_flags = pe_find_renamed\|pe_find_any;
	return TRUE;
	}

	gboolean
	timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.timeout_ms = crm_get_msec(optarg);
	return TRUE;
	}

	gboolean
	validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_execute_agent);
	if (options.operation) {
	g_free(options.operation);
	}
	options.operation = g_strdup(option_name + 2); // skip "--"
	options.find_flags = pe_find_renamed\|pe_find_anon;
	options.override_params = crm_str_table_new();
	return TRUE;
	}

	gboolean
	restart_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_restart);
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	gboolean
	wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_wait);
	options.require_resource = FALSE;
	options.require_dataset = FALSE;
	return TRUE;
	}

	gboolean
	why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.require_resource = FALSE;
	SET_COMMAND(cmd_why);
	options.find_flags = pe_find_renamed\|pe_find_anon;
	return TRUE;
	}

	static int
	ban_or_move(pcmk__output_t out, pe_resource_t rsc, const char *move_lifetime,
	crm_exit_t *exit_code)
	{
	int rc = pcmk_rc_ok;
	pe_node_t *current = NULL;
	unsigned int nactive = 0;

	CRM_CHECK(rsc != NULL, return EINVAL);

	current = pe__find_active_requires(rsc, &nactive);

	if (nactive == 1) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
	cib_conn, options.cib_options, options.promoted_role_only);

	} else if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	int count = 0;
	GListPtr iter = NULL;

	current = NULL;
	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	count++;
	current = pe__current_node(child);
	}
	}

	if(count == 1 && current) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime, NULL,
	cib_conn, options.cib_options, options.promoted_role_only);

	} else {
	rc = EINVAL;
	*exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"Resource '%s' not moved: active in %d locations (promoted in %d).\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node."
	"To prevent '%s' from being promoted at a specific "
	"location, specify a node and the master option.",
	options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
	}

	} else {
	rc = EINVAL;
	*exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"Resource '%s' not moved: active in %d locations.\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node.",
	options.rsc_id, nactive, options.rsc_id);
	}

	return rc;
	}

	static void
	cleanup(pcmk__output_t out, pe_resource_t rsc)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Erasing failures of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, options.operation,
	options.interval_spec, TRUE, data_set, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, cib_conn, rsc);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	clear_constraints(pcmk__output_t out, xmlNodePtr cib_xml_copy)
	{
	GListPtr before = NULL;
	GListPtr after = NULL;
	GListPtr remaining = NULL;
	GListPtr ele = NULL;
	pe_node_t *dest = NULL;
	int rc = pcmk_rc_ok;

	if (!out->is_quiet(out)) {
	before = build_constraint_list(data_set->input);
	}

	if (options.clear_expired) {
	rc = cli_resource_clear_all_expired(data_set->input, cib_conn, options.cib_options,
	options.rsc_id, options.host_uname,
	options.promoted_role_only);

	} else if (options.host_uname) {
	dest = pe_find_node(data_set->nodes, options.host_uname);
	if (dest == NULL) {
	rc = pcmk_rc_node_unknown;
	if (!out->is_quiet(out)) {
	g_list_free(before);
	}
	return rc;
	}
	rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
	cib_conn, options.cib_options, TRUE, options.force);

	} else {
	rc = cli_resource_clear(options.rsc_id, NULL, data_set->nodes,
	cib_conn, options.cib_options, TRUE, options.force);
	}

	if (!out->is_quiet(out)) {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not get modified CIB: %s\n", pcmk_strerror(rc));
	g_list_free(before);
	return rc;
	}

	data_set->input = *cib_xml_copy;
	cluster_status(data_set);

	after = build_constraint_list(data_set->input);
	remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);

	for (ele = remaining; ele != NULL; ele = ele->next) {
	out->info(out, "Removing constraint: %s", (char *) ele->data);
	}

	g_list_free(before);
	g_list_free(after);
	g_list_free(remaining);
	}

	return rc;
	}

	static int
	delete()
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (options.rsc_type == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"You need to specify a resource type with -t");
	return rc;
	}

	msg_data = create_xml_node(NULL, options.rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);

	rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
	options.cib_options);
	rc = pcmk_legacy2rc(rc);
	free_xml(msg_data);
	return rc;
	}

	static int
	list_agents(pcmk__output_t out, const char agent_spec, crm_exit_t *exit_code)
	{
	int rc = pcmk_rc_ok;
	char *provider = strchr(agent_spec, ':');
	lrmd_t *lrmd_conn = lrmd_api_new();
	lrmd_list_t *list = NULL;

	if (provider) {
	*provider++ = 0;
	}

	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, agent_spec, provider);

	if (rc > 0) {
	rc = out->message(out, "agents-list", list, agent_spec, provider);
	} else {
	rc = pcmk_rc_error;
	}

	if (rc != pcmk_rc_ok) {
	*exit_code = CRM_EX_NOSUCH;
	if (provider == NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"No agents found for standard '%s'", agent_spec);
	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"No agents found for standard '%s' and provider '%s'",
	agent_spec, provider);
	}
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static int
	list_providers(pcmk__output_t out, const char agent_spec, crm_exit_t *exit_code)
	{
	int rc;
	const char *text = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();
	lrmd_list_t *list = NULL;

	switch (options.rsc_cmd) {
	case cmd_list_alternatives:
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);

	if (rc > 0) {
	rc = out->message(out, "alternatives-list", list, agent_spec);
	} else {
	rc = pcmk_rc_error;
	}

	text = "OCF providers";
	break;
	case cmd_list_standards:
	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);

	if (rc > 0) {
	rc = out->message(out, "standards-list", list);
	} else {
	rc = pcmk_rc_error;
	}

	text = "standards";
	break;
	case cmd_list_providers:
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, agent_spec, &list);

	if (rc > 0) {
	rc = out->message(out, "providers-list", list, agent_spec);
	} else {
	rc = pcmk_rc_error;
	}

	text = "OCF providers";
	break;
	default:
	*exit_code = CRM_EX_SOFTWARE;
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code, "Bug");
	lrmd_api_delete(lrmd_conn);
	return pcmk_rc_error;
	}

	if (rc != pcmk_rc_ok) {
	if (agent_spec != NULL) {
	*exit_code = CRM_EX_NOSUCH;
	rc = pcmk_rc_error;
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"No %s found for %s", text, agent_spec);

	} else {
	*exit_code = CRM_EX_NOSUCH;
	rc = pcmk_rc_error;
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"No %s found", text);
	}
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static int
	populate_working_set(xmlNodePtr *cib_xml_copy)
	{
	int rc = pcmk_rc_ok;

	if (options.xml_file != NULL) {
	*cib_xml_copy = filename2xml(options.xml_file);
	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	}

	if(rc != pcmk_rc_ok) {
	return rc;
	}

	/* Populate the working set instance */
	data_set = pe_new_working_set();
	if (data_set == NULL) {
	rc = ENOMEM;
	return rc;
	}

	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);
	rc = update_working_set_xml(data_set, cib_xml_copy);
	if (rc == pcmk_rc_ok) {
	cluster_status(data_set);
	}
	return rc;
	}

	static int
	refresh(pcmk__output_t *out)
	{
	int rc = pcmk_rc_ok;
	const char *router_node = options.host_uname;
	int attr_options = pcmk__node_attr_none;

	if (options.host_uname) {
	pe_node_t *node = pe_find_node(data_set->nodes, options.host_uname);

	if (pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"No cluster connection to Pacemaker Remote node %s detected",
	options.host_uname);
	return rc;
	}
	router_node = node->details->uname;
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	options.host_uname? options.host_uname : "all nodes");
	rc = pcmk_rc_ok;
	return rc;
	}

	crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");

	rc = pcmk__node_attr_request_clear(NULL, options.host_uname,
	NULL, NULL, NULL,
	NULL, attr_options);

	if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
	router_node) == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}

	return rc;
	}

	static void
	refresh_resource(pcmk__output_t out, pe_resource_t rsc)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Re-checking the state of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(out, controld_api, options.host_uname, rsc, NULL,
	0, FALSE, data_set, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, cib_conn, rsc);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	set_property()
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (pcmk__str_empty(options.rsc_type)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Must specify -t with resource type");
	rc = ENXIO;
	return rc;

	} else if (pcmk__str_empty(options.prop_value)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Must supply -v with new value");
	rc = EINVAL;
	return rc;
	}

	CRM_LOG_ASSERT(options.prop_name != NULL);

	msg_data = create_xml_node(NULL, options.rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, options.rsc_id);
	crm_xml_add(msg_data, options.prop_name, options.prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
	options.cib_options);
	rc = pcmk_legacy2rc(rc);
	free_xml(msg_data);

	return rc;
	}

	static int
	show_metadata(pcmk__output_t out, const char agent_spec, crm_exit_t *exit_code)
	{
	int rc = pcmk_rc_ok;
	char *standard = NULL;
	char *provider = NULL;
	char *type = NULL;
	char *metadata = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();

	rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
	provider, type,
	&metadata, 0);
	rc = pcmk_legacy2rc(rc);

	if (metadata) {
	out->output_xml(out, "metadata", metadata);
	} else {
	*exit_code = crm_errno2exit(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, *exit_code,
	"Metadata query for %s failed: %s",
	agent_spec, pcmk_rc_str(rc));
	}
	} else {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"'%s' is not a valid agent specification", agent_spec);
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static void
	validate_cmdline(crm_exit_t *exit_code)
	{
	// -r cannot be used with any of --class, --agent, or --provider
	if (options.rsc_id != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--resource cannot be used with --class, --agent, and --provider");

	// If --class, --agent, or --provider are given, --validate must also be given.
	} else if (options.rsc_cmd != cmd_execute_agent) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--class, --agent, and --provider require --validate");

	// Not all of --class, --agent, and --provider need to be given. Not all
	// classes support the concept of a provider. Check that what we were given
	// is valid.
	} else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
	if (options.v_provider != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"stonith does not support providers");

	} else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"%s is not a known stonith agent", options.v_agent ? options.v_agent : "");
	}

	} else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"%s:%s:%s is not a known resource",
	options.v_class ? options.v_class : "",
	options.v_provider ? options.v_provider : "",
	options.v_agent ? options.v_agent : "");
	}

	if (error == NULL) {
	if (options.validate_options == NULL) {
	options.validate_options = crm_str_table_new();
	}
	*exit_code = cli_resource_execute_from_params(out, "test", options.v_class, options.v_provider, options.v_agent,
	"validate-all", options.validate_options,
	options.override_params, options.timeout_ms,
	options.resource_verbose, options.force);
	}
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },
	{ "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
	"Resource ID",
	"ID" },
	{ G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
	NULL,
	NULL },

	{ NULL }
	};

	const char *description = "Examples:\n\n"
	"List the available OCF agents:\n\n"
	"\t# crm_resource --list-agents ocf\n\n"
	"List the available OCF agents from the linux-ha project:\n\n"
	"\t# crm_resource --list-agents ocf:heartbeat\n\n"
	"Move 'myResource' to a specific node:\n\n"
	"\t# crm_resource --resource myResource --move --node altNode\n\n"
	"Allow (but not force) 'myResource' to move back to its original "
	"location:\n\n"
	"\t# crm_resource --resource myResource --clear\n\n"
	"Stop 'myResource' (and anything that depends on it):\n\n"
	"\t# crm_resource --resource myResource --set-parameter target-role "
	"--meta --parameter-value Stopped\n\n"
	"Tell the cluster not to manage 'myResource' (the cluster will not "
	"attempt to start or stop the\n"
	"resource under any circumstances; useful when performing maintenance "
	"tasks on a resource):\n\n"
	"\t# crm_resource --resource myResource --set-parameter is-managed "
	"--meta --parameter-value false\n\n"
	"Erase the operation history of 'myResource' on 'aNode' (the cluster "
	"will 'forget' the existing\n"
	"resource state, including any errors, and attempt to recover the"
	"resource; useful when a resource\n"
	"had failed permanently and has been repaired by an administrator):\n\n"
	"\t# crm_resource --resource myResource --cleanup --node aNode\n\n";

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	g_option_context_set_description(context, description);

	/* Add the -Q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show query help", query_entries);
	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command help", command_entries);
	pcmk__add_arg_group(context, "locations", "Locations:",
	"Show location help", location_entries);
	pcmk__add_arg_group(context, "validate", "Validate:",
	"Show validate help", validate_entries);
	pcmk__add_arg_group(context, "advanced", "Advanced:",
	"Show advanced option help", advanced_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	xmlNode *cib_xml_copy = NULL;
	pe_resource_t *rsc = NULL;

	int rc = pcmk_rc_ok;

	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	GOptionContext *context = NULL;
	GOptionGroup *output_group = NULL;
	gchar **processed_args = NULL;

	context = build_arg_context(args, &output_group);
	pcmk__register_formats(output_group, formats);
	crm_log_cli_init("crm_resource");

	processed_args = pcmk__cmdline_preproc(argv, "GINSTdginpstuv");

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	for (int i = 0; i < args->verbosity; i++) {
	crm_bump_log_level(argc, argv);
	}

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	fprintf(stderr, "Error creating output format %s: %s\n",
	args->output_ty, pcmk_rc_str(rc));
	exit_code = CRM_EX_ERROR;
	goto done;
	}

	options.resource_verbose = args->verbosity;
	out->quiet = args->quiet;

	crm_log_args(argc, argv);

	if (options.host_uname) {
	crm_trace("Option host => %s", options.host_uname);
	}

	// If the user didn't explicitly specify a command, list resources
	if (options.rsc_cmd == cmd_none) {
	options.rsc_cmd = cmd_list_resources;
	options.require_resource = FALSE;
	}

	// --expired without --clear/-U doesn't make sense
	if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "--expired requires --clear or -U");
	goto done;
	}

	if ((options.remainder != NULL) && (options.override_params != NULL)) {
	// Commands that use positional arguments will create override_params
	for (gchar *s = options.remainder; s; s++) {
	char name = calloc(1, strlen(s));
	char value = calloc(1, strlen(s));
	int rc = sscanf(*s, "%[^=]=%s", name, value);

	if (rc == 2) {
	g_hash_table_replace(options.override_params, name, value);

	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Error parsing '%s' as a name=value pair",
	argv[optind]);
	free(value);
	free(name);
	goto done;
	}
	}

	} else if (options.remainder != NULL) {
	gchar **strv = NULL;
	gchar *msg = NULL;
	int i = 1;
	int len = 0;

	for (gchar *s = options.remainder; s; s++) {
	len++;
	}

	CRM_ASSERT(len > 0);

	strv = calloc(len, sizeof(char *));
	strv[0] = strdup("non-option ARGV-elements:");

	for (gchar *s = options.remainder; s; s++) {
	strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
	i++;
	}

	msg = g_strjoinv("", strv);
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "%s", msg);

	for(i = 0; i < len; i++) {
	free(strv[i]);
	}

	g_free(msg);
	g_free(strv);
	goto done;
	}

	if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	/* Kind of a hack to display XML lists using a real tag instead of <list>. This just
	* saves from having to write custom messages to build the lists around all these things
	*/
	if (options.rsc_cmd == cmd_list_resources \|\| options.rsc_cmd == cmd_query_xml \|\|
	options.rsc_cmd == cmd_query_raw_xml \|\| options.rsc_cmd == cmd_list_active_ops \|\|
	options.rsc_cmd == cmd_list_all_ops) {
	pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
	} else {
	pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
	}
	} else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
	if (options.rsc_cmd == cmd_colocations \|\| options.rsc_cmd == cmd_colocations_deep \|\|
	options.rsc_cmd == cmd_list_resources) {
	pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
	}
	}

	pe__register_messages(out);
	crm_resource_register_messages(out);
	lrmd__register_messages(out);
	pcmk__register_lib_messages(out);

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (optind > argc) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Invalid option(s) supplied, use --help for valid usage");
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	// Sanity check validating from command line parameters. If everything checks out,
	// go ahead and run the validation. This way we don't need a CIB connection.
	if (options.validate_cmdline) {
	validate_cmdline(&exit_code);
	goto done;
	} else if (options.validate_options != NULL) {
	// @COMPAT @TODO error out here when we can break backward compatibility
	g_hash_table_destroy(options.validate_options);
	options.validate_options = NULL;
	}

	if (error != NULL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (options.force) {
	crm_debug("Forcing...");
	cib__set_call_options(options.cib_options, crm_system_name,
	cib_quorum_override);
	}

	if (options.require_resource && !options.rsc_id) {
	rc = ENXIO;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Must supply a resource id with -r");
	goto done;
	}

	if (options.find_flags && options.rsc_id) {
	options.require_dataset = TRUE;
	}

	// Establish a connection to the CIB if needed
	if (options.require_cib) {
	cib_conn = cib_new();
	if ((cib_conn == NULL) \|\| (cib_conn->cmds == NULL)) {
	rc = pcmk_rc_error;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_DISCONNECT,
	"Could not create CIB connection");
	goto done;
	}
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not connect to the CIB: %s", pcmk_rc_str(rc));
	goto done;
	}
	}

	/* Populate working set from XML file if specified or CIB query otherwise */
	if (options.require_dataset) {
	rc = populate_working_set(&cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	goto done;
	}
	}

	// If command requires that resource exist if specified, find it
	if (options.find_flags && options.rsc_id) {
	rsc = pe_find_resource_with_flags(data_set->resources, options.rsc_id,
	options.find_flags);
	if (rsc == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Resource '%s' not found", options.rsc_id);
	goto done;
	}
	}

	// Establish a connection to the controller if needed
	if (options.require_crmd) {
	rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Error connecting to the controller: %s", pcmk_rc_str(rc));
	goto done;
	}
	pcmk_register_ipc_callback(controld_api, controller_event_callback,
	NULL);
	rc = pcmk_connect_ipc(controld_api, pcmk_ipc_dispatch_main);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Error connecting to the controller: %s", pcmk_rc_str(rc));
	goto done;
	}
	}

	switch (options.rsc_cmd) {
	case cmd_list_resources: {
	GListPtr all = NULL;
	all = g_list_prepend(all, strdup("*"));
	rc = out->message(out, "resource-list", data_set,
	pe_print_rsconly \| pe_print_pending,
	FALSE, TRUE, FALSE, TRUE, all, all, FALSE);
	g_list_free_full(all, free);

	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}
	break;
	}

	case cmd_list_instances:
	rc = out->message(out, "resource-names-list", data_set->resources);

	if (rc != pcmk_rc_ok) {
	rc = ENXIO;
	}

	break;

	case cmd_list_standards:
	case cmd_list_providers:
	case cmd_list_alternatives:
	rc = list_providers(out, options.agent_spec, &exit_code);
	break;

	case cmd_list_agents:
	rc = list_agents(out, options.agent_spec, &exit_code);
	break;

	case cmd_metadata:
	rc = show_metadata(out, options.agent_spec, &exit_code);
	break;

	case cmd_restart:
	/* We don't pass data_set because rsc needs to stay valid for the
	* entire lifetime of cli_resource_restart(), but it will reset and
	* update the working set multiple times, so it needs to use its own
	* copy.
	*/
	rc = cli_resource_restart(out, rsc, options.host_uname,
	options.move_lifetime, options.timeout_ms,
	cib_conn, options.cib_options,
	options.promoted_role_only,
	options.force);
	break;

	case cmd_wait:
	rc = wait_till_stable(out, options.timeout_ms, cib_conn);
	break;

	case cmd_execute_agent:
	exit_code = cli_resource_execute(out, rsc, options.rsc_id,
	options.operation,
	options.override_params,
	options.timeout_ms, cib_conn,
	data_set, options.resource_verbose,
	options.force);
	break;

	case cmd_colocations:
	rc = out->message(out, "stacks-constraints", rsc, data_set, false);
	break;

	case cmd_colocations_deep:
	rc = out->message(out, "stacks-constraints", rsc, data_set, true);
	break;

	case cmd_cts:
	rc = pcmk_rc_ok;

	for (GList *lpc = data_set->resources; lpc != NULL;
	lpc = lpc->next) {

	rsc = (pe_resource_t *) lpc->data;
	cli_resource_print_cts(out, rsc);
	}

	cli_resource_print_cts_constraints(out, data_set);
	break;

	case cmd_fail:
	rc = cli_resource_fail(out, controld_api, options.host_uname,
	options.rsc_id, data_set);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	break;

	case cmd_list_active_ops:
	rc = cli_resource_print_operations(out, options.rsc_id,
	options.host_uname, TRUE,
	data_set);
	break;

	case cmd_list_all_ops:
	rc = cli_resource_print_operations(out, options.rsc_id,
	options.host_uname, FALSE,
	data_set);
	break;

	case cmd_locate: {
	GListPtr resources = cli_resource_search(out, rsc, options.rsc_id, data_set);
	- rc = out->message(out, "resource-search", resources, rsc, options.rsc_id);
	+ rc = out->message(out, "resource-search-list", resources, rsc, options.rsc_id);
	break;
	}

	case cmd_query_xml:
	rc = cli_resource_print(out, rsc, data_set, TRUE);
	break;

	case cmd_query_raw_xml:
	rc = cli_resource_print(out, rsc, data_set, FALSE);
	break;

	case cmd_why:
	{
	pe_node_t *dest = NULL;

	if (options.host_uname) {
	dest = pe_find_node(data_set->nodes, options.host_uname);
	if (dest == NULL) {
	rc = pcmk_rc_node_unknown;
	goto done;
	}
	}
	- out->message(out, "resource-why", cib_conn, data_set->resources, rsc, dest);
	+ out->message(out, "resource-reasons-list", cib_conn, data_set->resources, rsc, dest);
	rc = pcmk_rc_ok;
	}
	break;

	case cmd_clear:
	rc = clear_constraints(out, &cib_xml_copy);
	break;

	case cmd_move:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime, &exit_code);
	} else {
	rc = cli_resource_move(out, rsc, options.rsc_id, options.host_uname,
	options.move_lifetime, cib_conn,
	options.cib_options, data_set,
	options.promoted_role_only,
	options.force);
	}
	break;

	case cmd_ban:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime, &exit_code);
	} else {
	pe_node_t *dest = pe_find_node(data_set->nodes,
	options.host_uname);

	if (dest == NULL) {
	rc = pcmk_rc_node_unknown;
	goto done;
	}
	rc = cli_resource_ban(out, options.rsc_id, dest->details->uname,
	options.move_lifetime, NULL, cib_conn,
	options.cib_options,
	options.promoted_role_only);
	}
	break;

	case cmd_get_property:
	- rc = out->message(out, "property", rsc, options.prop_name);
	+ rc = out->message(out, "property-list", rsc, options.prop_name);
	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}

	break;

	case cmd_set_property:
	rc = set_property();
	break;

	case cmd_get_param: {
	unsigned int count = 0;
	GHashTable *params = NULL;
	pe_node_t *current = pe__find_active_on(rsc, &count, NULL);

	if (count > 1) {
	out->err(out, "%s is active on more than one node,"
	" returning the default value for %s", rsc->id, crm_str(options.prop_name));
	current = NULL;
	}

	params = crm_str_table_new();

	if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	get_rsc_attributes(params, rsc, current, data_set);

	} else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	/* No need to redirect to the parent */
	get_meta_attributes(params, rsc, current, data_set);

	} else {
	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
	NULL, FALSE, data_set);
	}

	crm_debug("Looking up %s in %s", options.prop_name, rsc->id);
	- rc = out->message(out, "attribute", rsc, options.prop_name, params);
	+ rc = out->message(out, "attribute-list", rsc, options.prop_name, params);
	g_hash_table_destroy(params);
	break;
	}

	case cmd_set_param:
	if (pcmk__str_empty(options.prop_value)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"You need to supply a value with the -v option");
	rc = EINVAL;
	goto done;
	}

	/* coverity[var_deref_model] False positive */
	rc = cli_resource_update_attribute(out, rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name,
	options.prop_value,
	options.recursive, cib_conn,
	options.cib_options, data_set,
	options.force);
	break;

	case cmd_delete_param:
	/* coverity[var_deref_model] False positive */
	rc = cli_resource_delete_attribute(out, rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name, cib_conn,
	options.cib_options, data_set,
	options.force);
	break;

	case cmd_cleanup:
	if (rsc == NULL) {
	rc = cli_cleanup_all(out, controld_api, options.host_uname,
	options.operation, options.interval_spec,
	data_set);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	} else {
	cleanup(out, rsc);
	}
	break;

	case cmd_refresh:
	if (rsc == NULL) {
	rc = refresh(out);
	} else {
	refresh_resource(out, rsc);
	}
	break;

	case cmd_delete:
	rc = delete();
	break;

	default:
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_SOFTWARE,
	"Unimplemented command: %d", (int) options.rsc_cmd);
	break;
	}

	done:
	if (rc != pcmk_rc_ok) {
	if (rc == pcmk_rc_no_quorum) {
	g_prefix_error(&error, "To ignore quorum, use the force option.\n");
	}

	if (error != NULL) {
	char *msg = crm_strdup_printf("%s\nError performing operation: %s",
	error->message, pcmk_rc_str(rc));
	g_clear_error(&error);
	g_set_error(&error, PCMK__RC_ERROR, rc, "%s", msg);
	free(msg);
	} else {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Error performing operation: %s", pcmk_rc_str(rc));
	}

	if (exit_code == CRM_EX_OK) {
	exit_code = pcmk_rc2exitc(rc);
	}
	}

	g_free(options.host_uname);
	g_free(options.interval_spec);
	g_free(options.move_lifetime);
	g_free(options.operation);
	g_free(options.prop_id);
	free(options.prop_name);
	g_free(options.prop_set);
	g_free(options.prop_value);
	g_free(options.rsc_id);
	g_free(options.rsc_type);
	free(options.agent_spec);
	free(options.v_agent);
	free(options.v_class);
	free(options.v_provider);
	g_free(options.xml_file);
	g_strfreev(options.remainder);

	if (options.override_params != NULL) {
	g_hash_table_destroy(options.override_params);
	}

	/* options.validate_options does not need to be destroyed here. See the
	* comments in cli_resource_execute_from_params.
	*/

	g_strfreev(processed_args);
	g_option_context_free(context);

	return bye(exit_code);
	}
	diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
	index 4cb78b53d9..89d6172dd3 100644
	--- a/tools/crm_resource_print.c
	+++ b/tools/crm_resource_print.c
	@@ -1,575 +1,574 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_resource.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/output_internal.h>

	#define cons_string(x) x?x:"NA"
	void
	cli_resource_print_cts_constraints(pcmk__output_t out, pe_working_set_t data_set)
	{
	xmlNode *xml_obj = NULL;
	xmlNode *lifetime = NULL;
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	for (xml_obj = pcmk__xe_first_child(cib_constraints); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);

	if (id == NULL) {
	continue;
	}

	// @COMPAT lifetime is deprecated
	lifetime = first_named_child(xml_obj, "lifetime");
	if (pe_evaluate_rules(lifetime, NULL, data_set->now, NULL) == FALSE) {
	continue;
	}

	if (!pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj), pcmk__str_casei)) {
	continue;
	}

	out->info(out, "Constraint %s %s %s %s %s %s %s",
	crm_element_name(xml_obj),
	cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
	cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
	}
	}

	void
	cli_resource_print_cts(pcmk__output_t out, pe_resource_t rsc)
	{
	GListPtr lpc = NULL;
	const char *host = NULL;
	bool needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	pe_node_t *node = pe__current_node(rsc);

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	needs_quorum = FALSE;
	} else {
	// @TODO check requires in resource meta-data and rsc_defaults
	}

	if (node != NULL) {
	host = node->details->uname;
	}

	out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx",
	crm_element_name(rsc->xml), rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	pe_resource_t child = (pe_resource_t ) lpc->data;

	cli_resource_print_cts(out, child);
	}
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print_operations(pcmk__output_t out, const char rsc_id,
	const char *host_uname, bool active,
	pe_working_set_t * data_set)
	{
	int rc = pcmk_rc_no_output;
	GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);

	if (!ops) {
	return rc;
	}

	out->begin_list(out, NULL, NULL, "Resource Operations");
	rc = pcmk_rc_ok;

	for (GListPtr lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;
	out->message(out, "node-and-op", data_set, xml_op);
	}

	out->end_list(out);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print(pcmk__output_t out, pe_resource_t rsc,
	pe_working_set_t *data_set, bool expanded)
	{
	unsigned int opts = pe_print_pending;
	GListPtr all = NULL;

	all = g_list_prepend(all, strdup("*"));

	out->begin_list(out, NULL, NULL, "Resource Config");
	out->message(out, crm_map_element_name(rsc->xml), opts, rsc, all, all);
	out->message(out, "resource-config", rsc, !expanded);
	out->end_list(out);

	g_list_free_full(all, free);
	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("attribute", "pe_resource_t ", "char ", "GHashTable *")
	+PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t ", "char ", "GHashTable *")
	static int
	-attribute_default(pcmk__output_t *out, va_list args) {
	+attribute_list_default(pcmk__output_t *out, va_list args) {
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	char attr = va_arg(args, char );
	GHashTable params = va_arg(args, GHashTable );

	const char *value = g_hash_table_lookup(params, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Attributes");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}

	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("attribute", "pe_resource_t ", "char ", "GHashTable *")
	+PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t ", "char ", "GHashTable *")
	static int
	-attribute_text(pcmk__output_t *out, va_list args) {
	+attribute_list_text(pcmk__output_t *out, va_list args) {
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	char attr = va_arg(args, char );
	GHashTable params = va_arg(args, GHashTable );

	const char *value = g_hash_table_lookup(params, attr);

	if (value != NULL) {
	out->info(out, "%s", value);
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}

	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("property", "pe_resource_t ", "char ")
	+PCMK__OUTPUT_ARGS("property-list", "pe_resource_t ", "char ")
	static int
	-property_default(pcmk__output_t *out, va_list args) {
	+property_list_default(pcmk__output_t *out, va_list args) {
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Properties");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	}

	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("property", "pe_resource_t ", "char ")
	+PCMK__OUTPUT_ARGS("property-list", "pe_resource_t ", "char ")
	static int
	-property_text(pcmk__output_t *out, va_list args) {
	+property_list_text(pcmk__output_t *out, va_list args) {
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->info(out, "%s", value);
	}

	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("resource-check", "resource_checks_t *")
	+PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	-resource_check_default(pcmk__output_t *out, va_list args) {
	+resource_check_list_default(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	pe_resource_t *parent = uber_parent(checks->rsc);
	int rc = pcmk_rc_no_output;
	bool printed = false;

	if (checks->flags != 0 \|\| checks->lock_node != NULL) {
	printed = true;
	out->begin_list(out, NULL, NULL, "Resource Checks");
	}

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
	parent->id);
	}

	if (checks->lock_node) {
	out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
	parent->id, checks->lock_node);
	}

	if (printed) {
	out->end_list(out);
	rc = pcmk_rc_ok;
	}

	return rc;
	}

	-PCMK__OUTPUT_ARGS("resource-check", "resource_checks_t *")
	+PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	-resource_check_xml(pcmk__output_t *out, va_list args) {
	+resource_check_list_xml(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	pe_resource_t *parent = uber_parent(checks->rsc);
	int rc = pcmk_rc_no_output;

	xmlNodePtr node = pcmk__output_create_xml_node(out, "check",
	"id", parent->id,
	NULL);

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	crm_xml_add(node, "remain_stopped", "true");
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	crm_xml_add(node, "promotable", "false");
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	crm_xml_add(node, "unmanaged", "true");
	}

	if (checks->lock_node) {
	crm_xml_add(node, "locked-to", checks->lock_node);
	}

	return rc;
	}

	-PCMK__OUTPUT_ARGS("resource-search", "GList ", "pe_resource_t ", "gchar *")
	+PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "pe_resource_t ", "gchar *")
	static int
	-resource_search_default(pcmk__output_t *out, va_list args)
	+resource_search_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	gchar requested_name = va_arg(args, gchar );

	bool printed = false;
	int rc = pcmk_rc_no_output;

	if (!out->is_quiet(out) && nodes == NULL) {
	out->err(out, "resource %s is NOT running", requested_name);
	return rc;
	}

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	pe_node_t node = (pe_node_t ) lpc->data;

	if (!printed) {
	out->begin_list(out, NULL, NULL, "Nodes");
	printed = true;
	rc = pcmk_rc_ok;
	}

	if (out->is_quiet(out)) {
	out->list_item(out, "node", "%s", node->details->uname);
	} else {
	const char *state = "";

	if (!pe_rsc_is_clone(rsc) && rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
	state = " Master";
	}
	out->list_item(out, "node", "resource %s is running on: %s%s",
	requested_name, node->details->uname, state);
	}
	}

	if (printed) {
	out->end_list(out);
	}

	return rc;
	}

	-
	-PCMK__OUTPUT_ARGS("resource-search", "GList ", "pe_resource_t ", "gchar *")
	+PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "pe_resource_t ", "gchar *")
	static int
	-resource_search_xml(pcmk__output_t *out, va_list args)
	+resource_search_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	gchar requested_name = va_arg(args, gchar );

	pcmk__output_xml_create_parent(out, "nodes",
	"resource", requested_name,
	NULL);

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	pe_node_t node = (pe_node_t ) lpc->data;
	xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out, "node", node->details->uname);

	if (!pe_rsc_is_clone(rsc) && rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
	crm_xml_add(sub_node, "state", "promoted");
	}
	}

	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("resource-why", "cib_t ", "GList ", "pe_resource_t *",
	+PCMK__OUTPUT_ARGS("resource-reasons-list", "cib_t ", "GList ", "pe_resource_t *",
	"pe_node_t *")
	static int
	-resource_why_default(pcmk__output_t *out, va_list args)
	+resource_reasons_list_default(pcmk__output_t *out, va_list args)
	{
	cib_t cib_conn = va_arg(args, cib_t );
	GList resources = va_arg(args, GList );
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	pe_node_t node = va_arg(args, pe_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	out->begin_list(out, NULL, NULL, "Resource Reasons");

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	out->list_item(out, "reason", "Resource %s is not running", rsc->id);
	} else {
	out->list_item(out, "reason", "Resource %s is running", rsc->id);
	}

	cli_resource_check(out, cib_conn, rsc);
	g_list_free(hosts);
	hosts = NULL;
	}

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	} else {
	out->list_item(out, "reason", "Resource %s is not running on host %s",
	rsc->id, host_uname);
	}

	cli_resource_check(out, cib_conn, rsc);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	cli_resource_check(out, cib_conn, rsc);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
	rsc->id, host_uname);
	cli_resource_check(out, cib_conn, rsc);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	out->list_item(out, "reason", "Resource %s is %srunning",
	rsc->id, (hosts? "" : "not "));
	cli_resource_check(out, cib_conn, rsc);
	g_list_free(hosts);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	-PCMK__OUTPUT_ARGS("resource-why", "cib_t ", "GList ", "pe_resource_t *",
	+PCMK__OUTPUT_ARGS("resource-reasons-list", "cib_t ", "GList ", "pe_resource_t *",
	"pe_node_t *")
	static int
	-resource_why_xml(pcmk__output_t *out, va_list args)
	+resource_reasons_list_xml(pcmk__output_t *out, va_list args)
	{
	cib_t cib_conn = va_arg(args, cib_t );
	GList resources = va_arg(args, GList );
	pe_resource_t rsc = va_arg(args, pe_resource_t );
	pe_node_t node = va_arg(args, pe_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, "reason", NULL);

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;

	rsc->fns->location(rsc, &hosts, TRUE);

	pcmk__output_xml_create_parent(out, "resource",
	"id", rsc->id,
	"running", pcmk__btoa(hosts != NULL),
	NULL);

	cli_resource_check(out, cib_conn, rsc);
	pcmk__output_xml_pop_parent(out);
	g_list_free(hosts);
	hosts = NULL;
	}

	pcmk__output_xml_pop_parent(out);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	crm_xml_add(xml_node, "running_on", host_uname);
	}

	cli_resource_check(out, cib_conn, rsc);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	"id", rsc->id,
	"running", "true",
	"host", host_uname,
	NULL);

	cli_resource_check(out, cib_conn, rsc);
	pcmk__output_xml_pop_parent(out);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	"id", rsc->id,
	"running", "false",
	"host", host_uname,
	NULL);

	cli_resource_check(out, cib_conn, rsc);
	pcmk__output_xml_pop_parent(out);
	}

	pcmk__output_xml_pop_parent(out);
	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	crm_xml_add(xml_node, "running", pcmk__btoa(hosts != NULL));
	cli_resource_check(out, cib_conn, rsc);
	g_list_free(hosts);
	}

	return pcmk_rc_ok;
	}

	static void
	add_resource_name(pcmk__output_t out, pe_resource_t rsc) {
	if (rsc->children == NULL) {
	out->list_item(out, "resource", "%s", rsc->id);
	} else {
	for (GListPtr lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	pe_resource_t child = (pe_resource_t ) lpc->data;
	add_resource_name(out, child);
	}
	}
	}

	PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
	static int
	resource_names(pcmk__output_t *out, va_list args) {
	GList resources = va_arg(args, GList );

	if (resources == NULL) {
	out->err(out, "NO resources configured\n");
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Names");

	for (GList *lpc = resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t rsc = (pe_resource_t ) lpc->data;
	add_resource_name(out, rsc);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	- { "attribute", "default", attribute_default },
	- { "attribute", "text", attribute_text },
	- { "property", "default", property_default },
	- { "property", "text", property_text },
	- { "resource-check", "default", resource_check_default },
	- { "resource-check", "xml", resource_check_xml },
	- { "resource-search", "default", resource_search_default },
	- { "resource-search", "xml", resource_search_xml },
	- { "resource-why", "default", resource_why_default },
	- { "resource-why", "xml", resource_why_xml },
	+ { "attribute-list", "default", attribute_list_default },
	+ { "attribute-list", "text", attribute_list_text },
	+ { "property-list", "default", property_list_default },
	+ { "property-list", "text", property_list_text },
	+ { "resource-check-list", "default", resource_check_list_default },
	+ { "resource-check-list", "xml", resource_check_list_xml },
	+ { "resource-search-list", "default", resource_search_list_default },
	+ { "resource-search-list", "xml", resource_search_list_xml },
	+ { "resource-reasons-list", "default", resource_reasons_list_default },
	+ { "resource-reasons-list", "xml", resource_reasons_list_xml },
	{ "resource-names-list", "default", resource_names },

	{ NULL, NULL, NULL }
	};

	void
	crm_resource_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index bbd8bc1e53..3a9feac600 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,1959 +1,1959 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_resource.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/xml_internal.h>

	resource_checks_t *
	cli_check_resource(pe_resource_t rsc, char role_s, char *managed)
	{
	pe_resource_t *parent = uber_parent(rsc);
	resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));

	if (role_s) {
	enum rsc_role_e role = text2role(role_s);

	if (role == RSC_ROLE_STOPPED) {
	rc->flags \|= rsc_remain_stopped;
	} else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
	role == RSC_ROLE_SLAVE) {
	rc->flags \|= rsc_unpromotable;
	}
	}

	if (managed && !crm_is_true(managed)) {
	rc->flags \|= rsc_unmanaged;
	}

	if (rsc->lock_node) {
	rc->lock_node = rsc->lock_node->details->uname;
	}

	rc->rsc = rsc;
	return rc;
	}

	GListPtr
	cli_resource_search(pcmk__output_t out, pe_resource_t rsc, const char *requested_name,
	pe_working_set_t *data_set)
	{
	GListPtr found = NULL;
	pe_resource_t *parent = uber_parent(rsc);

	if (pe_rsc_is_clone(rsc)) {
	for (GListPtr iter = rsc->children; iter != NULL; iter = iter->next) {
	GListPtr extra = ((pe_resource_t *) iter->data)->running_on;
	if (extra != NULL) {
	found = g_list_concat(found, extra);
	}
	}

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	for (GListPtr iter = parent->children; iter; iter = iter->next) {
	GListPtr extra = ((pe_resource_t *) iter->data)->running_on;
	if (extra != NULL) {
	found = g_list_concat(found, extra);
	}
	}

	} else if (rsc->running_on != NULL) {
	found = g_list_concat(found, rsc->running_on);
	}

	return found;
	}

	#define XPATH_MAX 1024

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int offset = 0;
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	char *xpath_string = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_string = calloc(1, XPATH_MAX);
	offset +=
	snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", get_object_path("resources"));

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);

	if (attr_set_type) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", attr_set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
	CRM_LOG_ASSERT(offset > 0);

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = EINVAL;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	bail:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t out, GList/ <pe_resource_t> / ** result,
	pe_resource_t * rsc, const char * rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if(pe_clone == rsc->variant) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pe_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pe_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if(rsc->parent && pe_clone == rsc->parent->variant) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	if (!out->is_quiet(out)) {
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	}
	return g_list_append(result, rsc);
	} else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
	pe_resource_t *child = rsc->children->data;

	if(child->variant == pe_native) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	if (!out->is_quiet(out)) {
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pcmk__output_t out, pe_resource_t rsc,
	const char requested_name, const char attr_set,
	const char attr_set_type, const char attr_id,
	const char attr_name, const char attr_value,
	gboolean recursive, cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	int rc = pcmk_rc_ok;
	static bool need_init = TRUE;

	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	GList/<pe_resource_t>/ resources = NULL;
	const char *common_attr_id = attr_id;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (force == FALSE) {
	rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
	out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
	uber_parent(rsc)->id, attr_name, local_attr_id);
	out->err(out, " Delete '%s' first or use the force option to override",
	local_attr_id);
	}
	free(local_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "update", force);
	}

	/* If either attr_set or attr_id is specified,
	* one clearly intends to modify a single resource.
	* It is the last item on the resource list.*/
	for(GList *gIter = (attr_set\|\|attr_id) ? g_list_last(resources) : resources
	; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	local_attr_id = NULL;
	local_attr_set = NULL;

	rsc = (pe_resource_t*)gIter->data;
	attr_id = common_attr_id;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == pcmk_rc_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != ENXIO) {
	free(lookup_id);
	free(local_attr_id);
	g_list_free(resources);
	return rc;

	} else {
	const char *tag = crm_element_name(rsc->xml);

	if (attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_strdup_printf("%s-%s", attr_set, attr_name);
	attr_id = local_attr_id;
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(local_attr_id);
	free(local_attr_set);

	if(recursive && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	GListPtr lpc = NULL;

	if(need_init) {
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	need_init = FALSE;
	unpack_constraints(cib_constraints, data_set);

	for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	pe_resource_t r = (pe_resource_t ) lpc->data;

	pe__clear_resource_flags(r, pe_rsc_allocating);
	}
	}

	crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
	pe__set_resource_flags(rsc, pe_rsc_allocating);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) lpc->data;
	pe_resource_t *peer = cons->rsc_lh;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (cons->score > 0 && !pcmk_is_set(peer->flags, pe_rsc_allocating)) {
	/* Don't get into colocation loops */
	crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
	cli_resource_update_attribute(out, peer, peer->id, NULL, attr_set_type,
	NULL, attr_name, attr_value, recursive,
	cib, cib_options, data_set, force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pcmk__output_t out, pe_resource_t rsc,
	const char requested_name, const char attr_set,
	const char attr_set_type, const char attr_id,
	const char attr_name, cib_t cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	int rc = pcmk_rc_ok;
	GList/<pe_resource_t>/ resources = NULL;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "delete", force);
	} else {
	resources = g_list_append(resources, rsc);
	}

	for(GList *gIter = resources; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *local_attr_id = NULL;

	rsc = (pe_resource_t*)gIter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == ENXIO) {
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	} else if (rc != pcmk_rc_ok) {
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(local_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk__output_t out, pcmk_ipc_api_t controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id, pe_working_set_t *data_set)
	{
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pe_native) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	if (host_uname == NULL) {
	out->err(out, "Please specify a node name");
	return EINVAL;

	} else {
	pe_node_t *node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(pe_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char host_uname, const char rsc_id,
	pe_working_set_t *data_set)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(out, controld_api, false, host_uname, rsc_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	const char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	interval_ms_s = crm_strdup_printf("%u",
	crm_parse_interval_spec(interval_spec));
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
	failed_id,
	pe_find_renamed\|pe_find_anon);

	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	/* not available until glib 2.32
	g_hash_table_add(rscs, (gpointer) failed_id);
	*/
	g_hash_table_insert(rscs, (gpointer) failed_id, (gpointer) failed_id);
	}

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(out, controld_api, node_name, failed_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(pe_resource_t rsc, const char operation,
	const char interval_spec, pe_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char host_uname, pe_resource_t rsc, const char *operation,
	const char *interval_spec, bool just_failures,
	pe_working_set_t *data_set, gboolean force)
	{
	int rc = pcmk_rc_ok;
	pe_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {
	GListPtr lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	pe_resource_t child = (pe_resource_t ) lpc->data;

	rc = cli_resource_delete(out, controld_api, host_uname, child, operation,
	interval_spec, just_failures, data_set,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GListPtr lpc = NULL;
	GListPtr nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(data_set->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pe_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(out, controld_api, node->details->uname,
	rsc, operation, interval_spec,
	just_failures, data_set, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, data_set);
	} else {
	rc = clear_rsc_history(out, controld_api, host_uname, rsc->id, data_set);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_strerror(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char operation,
	const char interval_spec, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pe_node_t *node = pe_find_node(data_set->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_strerror(rc));
	return rc;
	}
	} else {
	for (GList *iter = data_set->nodes; iter; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_strerror(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	int
	cli_resource_check(pcmk__output_t out, cib_t cib_conn, pe_resource_t *rsc)
	{
	char *role_s = NULL;
	char *managed = NULL;
	pe_resource_t *parent = uber_parent(rsc);
	int rc = pcmk_rc_no_output;
	resource_checks_t *checks = NULL;

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);

	checks = cli_check_resource(rsc, role_s, managed);

	if (checks->flags != 0 \|\| checks->lock_node != NULL) {
	- rc = out->message(out, "resource-check", checks);
	+ rc = out->message(out, "resource-check-list", checks);
	}

	free(role_s);
	free(managed);
	free(checks);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char host_uname, const char rsc_id,
	pe_working_set_t *data_set)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(out, controld_api, true, host_uname, rsc_id, data_set);
	}

	static GHashTable *
	generate_resource_params(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;

	if (!rsc) {
	crm_err("Resource does not exist in config");
	return NULL;
	}

	params = crm_str_table_new();
	meta = crm_str_table_new();
	combined = crm_str_table_new();

	get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
	get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);

	if (params) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	g_hash_table_destroy(params);
	}

	if (meta) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pe_resource_t rsc, const char host)
	{
	bool found = TRUE;
	GListPtr hIter = NULL;
	GListPtr hosts = NULL;

	if(rsc == NULL) {
	return FALSE;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pe_node_t node = (pe_node_t ) hIter->data;

	if(strcmp(host, node->details->uname) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	} else if(strcmp(host, node->details->id) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if(host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = FALSE;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = FALSE;
	}

	done:

	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pe_resource_t rsc = (pe_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pe_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as working set input
	*
	* This also updates the working set timestamp to the current time.
	*
	* \param[in] data_set Working set instance to update
	* \param[in] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_working_set_xml(pe_working_set_t data_set, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	data_set->input = *xml;
	data_set->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update a working set's XML input based on a CIB query
	*
	* \param[in] data_set Data set instance to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->input and data_set->now.
	*/
	static int
	update_working_set_from_cib(pcmk__output_t out, pe_working_set_t data_set,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
	return rc;
	}
	rc = update_working_set_xml(data_set, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(pcmk__output_t out, cib_t cib, pe_working_set_t * data_set,
	bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pe_reset_working_set(data_set);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto cleanup;
	}

	rc = write_xml_file(data_set->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	pcmk__schedule_actions(data_set, data_set->input, NULL);
	run_simulation(data_set, shadow_cib, NULL, TRUE);
	rc = update_dataset(out, shadow_cib, data_set, FALSE);

	} else {
	cluster_status(data_set);
	}

	cleanup:
	/* Do not free data_set->input here, we need rsc->xml to be valid later on */
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	static int
	max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc)
	{
	int delay = 0;
	int max_delay = 0;

	if(rsc && rsc->children) {
	GList *iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;

	delay = max_delay_for_resource(data_set, child);
	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
	max_delay = delay;
	}
	}

	} else if(rsc) {
	char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
	pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
	const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);

	max_delay = value? (int) crm_parse_ll(value, NULL) : -1;
	pe_free_action(stop);
	}


	return max_delay;
	}

	static int
	max_delay_in(pe_working_set_t * data_set, GList *resources)
	{
	int max_delay = 0;
	GList *item = NULL;

	for (item = resources; item != NULL; item = item->next) {
	int delay = 0;
	pe_resource_t rsc = pe_find_resource(data_set->resources, (const char )item->data);

	delay = max_delay_for_resource(data_set, rsc);

	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
	max_delay = delay;
	}
	}

	return 5 + (max_delay / 1000);
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(resource_is_running_on((r), (h)) == FALSE))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in] rsc The resource to restart
	* \param[in] host The host to restart the resource on (or NULL for all)
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but a two-second
	* granularity is actually used; if 0, a timeout will be
	* calculated based on the resource timeout)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, const char *host,
	const char move_lifetime, int timeout_ms, cib_t cib,
	int cib_options, gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = FALSE;
	char *rsc_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pe_working_set_t *data_set = NULL;

	if(resource_is_running_on(rsc, host) == FALSE) {
	const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
	if(host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	rsc_id = strdup(rsc->id);
	if ((pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) && host) {
	stop_via_ban = TRUE;
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	crm_perror(LOG_ERR, "Could not allocate working set");
	rc = ENOMEM;
	goto done;
	}
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);
	rc = update_dataset(out, cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, data_set->resources);
	current_active = get_active_resources(host, data_set->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
	cib_options, promoted_role_only);

	} else {
	/* Stop the resource by setting target-role to Stopped.
	* Remember any existing target-role so we can restore it later
	* (though it only makes any difference if it's Slave).
	*/
	char *lookup_id = clone_strip(rsc->id);

	find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
	NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
	free(lookup_id);
	rc = cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	RSC_STOPPED, FALSE, cib, cib_options,
	data_set, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	goto done;
	}

	rc = update_dataset(out, cib, data_set, TRUE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, data_set->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(out, cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, data_set->resources);
	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->info(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, data_set, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
	cib_options, data_set, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	goto done;
	}

	if (target_active) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(out, cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	current_active = get_active_resources(NULL, data_set->resources);
	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->info(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, data_set, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(out, rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
	data_set, force);
	}

	done:
	if (list_delta) {
	g_list_free(list_delta);
	}
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (target_active && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	pe_free_working_set(data_set);
	return rc;
	}

	static inline bool action_is_pending(pe_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags, pe_action_optional\|pe_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pe_action_runnable)
	\|\| pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Return TRUE if any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return TRUE if any actions in the list are pending, FALSE otherwise
	*/
	static bool
	actions_are_pending(GListPtr actions)
	{
	GListPtr action;

	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t )action->data;
	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=0x%.8x)", a->uuid, a->flags);
	return TRUE;
	}
	}
	return FALSE;
	}

	static void
	print_pending_actions(pcmk__output_t *out, GListPtr actions)
	{
	GListPtr action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but one-second granularity
	* is actually used; if 0, a default will be used)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pe_working_set_t *data_set = NULL;
	int rc = pcmk_rc_ok;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	return ENOMEM;
	}
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);

	do {

	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
	} else {
	print_pending_actions(out, data_set->actions);
	pe_free_working_set(data_set);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(data_set);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(data_set);
	return rc;
	}
	pcmk__schedule_actions(data_set, data_set->input, NULL);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(data_set->config_hash,
	"dc-version");

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = TRUE;
	}
	}

	} while (actions_are_pending(data_set->actions));

	pe_free_working_set(data_set);
	return rc;
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force)
	{
	GHashTable *params_copy = NULL;
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	out->err(out, "Sorry, the %s option doesn't support %s resources yet",
	action, rsc_class);
	crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
	}

	/* If no timeout was provided, grab the default. */
	if (timeout_ms == 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	/* add meta_timeout env needed by some resource agents */
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	/* add crm_feature_set env needed by some resource agents */
	g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	/* resources_action_create frees the params hash table it's passed, but we
	* may need to reuse it in a second call to resources_action_create. Thus
	* we'll make a copy here so that gets freed and the original remains for
	* reuse.
	*/
	params_copy = crm_str_table_dup(params);

	op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
	timeout_ms, params_copy, 0);
	if (op == NULL) {
	/* Re-run with stderr enabled so we can display a sane error message */
	crm_enable_stderr(TRUE);
	params_copy = crm_str_table_dup(params);
	op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
	timeout_ms, params_copy, 0);

	/* Callers of cli_resource_execute expect that the params hash table will
	* be freed. That function uses this one, so for that reason and for
	* making the two act the same, we should free the hash table here too.
	*/
	g_hash_table_destroy(params);

	/* We know op will be NULL, but this makes static analysis happy */
	services_action_free(op);
	crm_exit(CRM_EX_DATAERR);
	return exit_code; // Never reached, but helps static analysis
	}

	setenv("HA_debug", resource_verbose > 0 ? "1" : "0", 1);
	if(resource_verbose > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);

	if (override_hash) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, override_hash);
	while (g_hash_table_iter_next(&iter, (gpointer ) & name, (gpointer ) & value)) {
	out->info(out, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
	rsc_name, name, value);
	g_hash_table_replace(op->params, strdup(name), strdup(value));
	}
	}

	if (services_action_sync(op)) {
	exit_code = op->rc;

	if (op->status == PCMK_LRM_OP_DONE) {
	out->info(out, "Operation %s for %s (%s:%s:%s) returned: '%s' (%d)",
	action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
	services_ocf_exitcode_str(op->rc), op->rc);
	} else {
	out->info(out, "Operation %s for %s (%s:%s:%s) failed: '%s' (%d)",
	action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
	services_lrm_status_str(op->status), op->status);
	}

	/* hide output for validate-all if not in verbose */
	if (resource_verbose == 0 && pcmk__str_eq(action, "validate-all", pcmk__str_casei))
	goto done;

	if (op->stdout_data \|\| op->stderr_data) {
	out->subprocess_output(out, op->rc, op->stdout_data, op->stderr_data);
	}
	} else {
	exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
	}

	done:
	services_action_free(op);
	/* See comment above about why we free params here. */
	g_hash_table_destroy(params);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pcmk__output_t out, pe_resource_t rsc,
	const char requested_name, const char rsc_action,
	GHashTable *override_hash, int timeout_ms,
	cib_t * cib, pe_working_set_t *data_set,
	int resource_verbose, gboolean force)
	{
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	const char *action = NULL;
	GHashTable *params = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = "validate-all";

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = "monitor";

	} else if (pcmk__str_eq(rsc_action, "force-stop", pcmk__str_casei)) {
	action = rsc_action+6;

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	action = rsc_action+6;

	if(pe_rsc_is_clone(rsc)) {
	GListPtr rscs = cli_resource_search(out, rsc, requested_name, data_set);
	if(rscs != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	action, rsc->id);
	out->err(out, "Try setting target-role=Stopped first or specifying "
	"the force option");
	return CRM_EX_UNSAFE;
	}
	}

	} else {
	action = rsc_action;
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if(rsc->variant == pe_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	params = generate_resource_params(rsc, data_set);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, action, data_set);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, action,
	params, override_hash, timeout_ms,
	resource_verbose, force);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(pcmk__output_t out, pe_resource_t rsc, const char *rsc_id,
	const char host_name, const char move_lifetime, cib_t *cib,
	int cib_options, pe_working_set_t *data_set,
	gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pe_node_t *current = NULL;
	pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
	bool cur_is_dest = FALSE;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pe_resource_t *p = uber_parent(rsc);

	if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	GListPtr iter = NULL;
	unsigned int master_count = 0;
	pe_node_t *master_node = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	rsc = child;
	master_node = pe__current_node(child);
	master_count++;
	}
	}
	if (promoted_role_only \|\| master_count) {
	count = master_count;
	current = master_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (current && (current->details == dest->details)) {
	cur_is_dest = TRUE;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active", dest->details->uname);
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, FALSE, force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only);

	crm_trace("%s%s now prefers node %s%s",
	rsc->id, promoted_role_only?" (master)":"", dest->details->uname, force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if(force && (cur_is_dest == FALSE)) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	NULL, cib, cib_options, promoted_role_only);

	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, dest->details->uname);
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Oct 29, 8:49 PM (1 d, 14 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 942621
Default Alt Text: (233 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions