No OneTemporary
Actions

Size

651 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/execd/cts-exec-helper.c b/daemons/execd/cts-exec-helper.c
	index ac214a7d8b..54756cc395 100644
	--- a/daemons/execd/cts-exec-helper.c
	+++ b/daemons/execd/cts-exec-helper.c
	@@ -1,626 +1,626 @@
	/*
	* Copyright 2012-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/lrmd.h>

	#define SUMMARY "cts-exec-helper - inject commands into the Pacemaker executor and watch for events"

	static int exec_call_id = 0;
	static gboolean start_test(gpointer user_data);
	static void try_connect(void);

	static char *key = NULL;
	static char *val = NULL;

	static struct {
	int verbose;
	int quiet;
	guint interval_ms;
	int timeout;
	int start_delay;
	int cancel_call_id;
	gboolean no_wait;
	gboolean is_running;
	gboolean no_connect;
	int exec_call_opts;
	const char *api_call;
	const char *rsc_id;
	const char *provider;
	const char *class;
	const char *type;
	const char *action;
	const char *listen;
	gboolean use_tls;
	lrmd_key_value_t *params;
	} options;

	static gboolean
	interval_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	return pcmk_parse_interval_spec(optarg,
	&options.interval_ms) == pcmk_rc_ok;
	}

	static gboolean
	notify_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "--notify-orig", "-n", NULL)) {
	options.exec_call_opts = lrmd_opt_notify_orig_only;
	} else if (pcmk__str_any_of(option_name, "--notify-changes", "-o", NULL)) {
	options.exec_call_opts = lrmd_opt_notify_changes_only;
	}

	return TRUE;
	}

	static gboolean
	param_key_val_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "--param-key", "-k", NULL)) {
	pcmk__str_update(&key, optarg);
	} else if (pcmk__str_any_of(option_name, "--param-val", "-v", NULL)) {
	pcmk__str_update(&val, optarg);
	}

	if (key != NULL && val != NULL) {
	options.params = lrmd_key_value_add(options.params, key, val);
	pcmk__str_update(&key, NULL);
	pcmk__str_update(&val, NULL);
	}

	return TRUE;
	}

	static GOptionEntry basic_entries[] = {
	{ "api-call", 'c', 0, G_OPTION_ARG_STRING, &options.api_call,
	"Directly relates to executor API functions",
	NULL },

	{ "is-running", 'R', 0, G_OPTION_ARG_NONE, &options.is_running,
	"Determine if a resource is registered and running",
	NULL },

	{ "listen", 'l', 0, G_OPTION_ARG_STRING, &options.listen,
	"Listen for a specific event string",
	NULL },

	{ "no-wait", 'w', 0, G_OPTION_ARG_NONE, &options.no_wait,
	"Make api call and do not wait for result",
	NULL },

	{ "notify-changes", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, notify_cb,
	"Only notify client changes to recurring operations",
	NULL },

	{ "notify-orig", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, notify_cb,
	"Only notify this client of the results of an API action",
	NULL },

	{ "tls", 'S', 0, G_OPTION_ARG_NONE, &options.use_tls,
	"Use TLS backend for local connection",
	NULL },

	{ NULL }
	};

	static GOptionEntry api_call_entries[] = {
	{ "action", 'a', 0, G_OPTION_ARG_STRING, &options.action,
	NULL, NULL },

	{ "cancel-call-id", 'x', 0, G_OPTION_ARG_INT, &options.cancel_call_id,
	NULL, NULL },

	{ "class", 'C', 0, G_OPTION_ARG_STRING, &options.class,
	NULL, NULL },

	{ "interval", 'i', 0, G_OPTION_ARG_CALLBACK, interval_cb,
	NULL, NULL },

	{ "param-key", 'k', 0, G_OPTION_ARG_CALLBACK, param_key_val_cb,
	NULL, NULL },

	{ "param-val", 'v', 0, G_OPTION_ARG_CALLBACK, param_key_val_cb,
	NULL, NULL },

	{ "provider", 'P', 0, G_OPTION_ARG_STRING, &options.provider,
	NULL, NULL },

	{ "rsc-id", 'r', 0, G_OPTION_ARG_STRING, &options.rsc_id,
	NULL, NULL },

	{ "start-delay", 's', 0, G_OPTION_ARG_INT, &options.start_delay,
	NULL, NULL },

	{ "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout,
	NULL, NULL },

	{ "type", 'T', 0, G_OPTION_ARG_STRING, &options.type,
	NULL, NULL },

	{ NULL }
	};

	static GMainLoop *mainloop = NULL;
	static lrmd_t *lrmd_conn = NULL;

	static char event_buf_v0[1024];

	static crm_exit_t
	test_exit(crm_exit_t exit_code)
	{
	lrmd_api_delete(lrmd_conn);
	return crm_exit(exit_code);
	}

	#define print_result(fmt, args...) \
	if (!options.quiet) { \
	printf(fmt "\n" , ##args); \
	}

	#define report_event(event) \
	snprintf(event_buf_v0, sizeof(event_buf_v0), "NEW_EVENT event_type:%s rsc_id:%s action:%s rc:%s op_status:%s", \
	lrmd_event_type2str(event->type), \
	event->rsc_id, \
	event->op_type ? event->op_type : "none", \
	services_ocf_exitcode_str(event->rc), \
	pcmk_exec_status_str(event->op_status)); \
	crm_info("%s", event_buf_v0);

	static void
	test_shutdown(int nsig)
	{
	lrmd_api_delete(lrmd_conn);
	lrmd_conn = NULL;
	}

	static void
	read_events(lrmd_event_data_t * event)
	{
	report_event(event);
	if (options.listen) {
	if (pcmk__str_eq(options.listen, event_buf_v0, pcmk__str_casei)) {
	print_result("LISTEN EVENT SUCCESSFUL");
	test_exit(CRM_EX_OK);
	}
	}

	if (exec_call_id && (event->call_id == exec_call_id)) {
	if (event->op_status == 0 && event->rc == 0) {
	print_result("API-CALL SUCCESSFUL for 'exec'");
	} else {
	print_result("API-CALL FAILURE for 'exec', rc:%d lrmd_op_status:%s",
	event->rc, pcmk_exec_status_str(event->op_status));
	test_exit(CRM_EX_ERROR);
	}

	if (!options.listen) {
	test_exit(CRM_EX_OK);
	}
	}
	}

	static gboolean
	timeout_err(gpointer data)
	{
	print_result("LISTEN EVENT FAILURE - timeout occurred, never found");
	test_exit(CRM_EX_TIMEOUT);
	return FALSE;
	}

	static void
	connection_events(lrmd_event_data_t * event)
	{
	int rc = event->connection_rc;

	if (event->type != lrmd_event_connect) {
	/* ignore */
	return;
	}

	if (!rc) {
	crm_info("Executor client connection established");
	start_test(NULL);
	return;
	} else {
	sleep(1);
	try_connect();
	crm_notice("Executor client connection failed");
	}
	}

	static void
	try_connect(void)
	{
	int tries = 10;
	static int num_tries = 0;
	int rc = 0;

	lrmd_conn->cmds->set_callback(lrmd_conn, connection_events);
	for (; num_tries < tries; num_tries++) {
	rc = lrmd_conn->cmds->connect_async(lrmd_conn, crm_system_name, 3000);

	if (!rc) {
	return; /* we'll hear back in async callback */
	}
	sleep(1);
	}

	print_result("API CONNECTION FAILURE");
	test_exit(CRM_EX_ERROR);
	}

	static gboolean
	start_test(gpointer user_data)
	{
	int rc = 0;

	if (!options.no_connect) {
	if (!lrmd_conn->cmds->is_connected(lrmd_conn)) {
	try_connect();
	/* async connect -- this function will get called back into */
	return 0;
	}
	}
	lrmd_conn->cmds->set_callback(lrmd_conn, read_events);

	if (options.timeout) {
	g_timeout_add(options.timeout, timeout_err, NULL);
	}

	if (!options.api_call) {
	return 0;
	}

	if (pcmk__str_eq(options.api_call, "exec", pcmk__str_casei)) {
	rc = lrmd_conn->cmds->exec(lrmd_conn,
	options.rsc_id,
	options.action,
	NULL,
	options.interval_ms,
	options.timeout,
	options.start_delay,
	options.exec_call_opts,
	options.params);

	if (rc > 0) {
	exec_call_id = rc;
	print_result("API-CALL 'exec' action pending, waiting on response");
	}

	} else if (pcmk__str_eq(options.api_call, "register_rsc", pcmk__str_casei)) {
	rc = lrmd_conn->cmds->register_rsc(lrmd_conn,
	options.rsc_id,
	options.class, options.provider, options.type, 0);
	} else if (pcmk__str_eq(options.api_call, "get_rsc_info", pcmk__str_casei)) {
	lrmd_rsc_info_t *rsc_info;

	rsc_info = lrmd_conn->cmds->get_rsc_info(lrmd_conn, options.rsc_id, 0);

	if (rsc_info) {
	print_result("RSC_INFO: id:%s class:%s provider:%s type:%s",
	rsc_info->id, rsc_info->standard,
	(rsc_info->provider? rsc_info->provider : "<none>"),
	rsc_info->type);
	lrmd_free_rsc_info(rsc_info);
	rc = pcmk_ok;
	} else {
	rc = -1;
	}
	} else if (pcmk__str_eq(options.api_call, "unregister_rsc", pcmk__str_casei)) {
	rc = lrmd_conn->cmds->unregister_rsc(lrmd_conn, options.rsc_id, 0);
	} else if (pcmk__str_eq(options.api_call, "cancel", pcmk__str_casei)) {
	rc = lrmd_conn->cmds->cancel(lrmd_conn, options.rsc_id, options.action,
	options.interval_ms);
	} else if (pcmk__str_eq(options.api_call, "metadata", pcmk__str_casei)) {
	char *output = NULL;

	rc = lrmd_conn->cmds->get_metadata(lrmd_conn,
	options.class,
	options.provider, options.type, &output, 0);
	if (rc == pcmk_ok) {
	print_result("%s", output);
	free(output);
	}
	} else if (pcmk__str_eq(options.api_call, "list_agents", pcmk__str_casei)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, options.class, options.provider);

	if (rc > 0) {
	print_result("%d agents found", rc);
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result("%s", iter->val);
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result("API_CALL FAILURE - no agents found");
	rc = -1;
	}
	} else if (pcmk__str_eq(options.api_call, "list_ocf_providers", pcmk__str_casei)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, options.type, &list);

	if (rc > 0) {
	print_result("%d providers found", rc);
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result("%s", iter->val);
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result("API_CALL FAILURE - no providers found");
	rc = -1;
	}

	} else if (pcmk__str_eq(options.api_call, "list_standards", pcmk__str_casei)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);

	if (rc > 0) {
	print_result("%d standards found", rc);
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result("%s", iter->val);
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result("API_CALL FAILURE - no providers found");
	rc = -1;
	}

	} else if (pcmk__str_eq(options.api_call, "get_recurring_ops", pcmk__str_casei)) {
	GList *op_list = NULL;
	GList *op_item = NULL;
	rc = lrmd_conn->cmds->get_recurring_ops(lrmd_conn, options.rsc_id, 0, 0,
	&op_list);

	for (op_item = op_list; op_item != NULL; op_item = op_item->next) {
	lrmd_op_info_t *op_info = op_item->data;

	print_result("RECURRING_OP: %s_%s_%s timeout=%sms",
	op_info->rsc_id, op_info->action,
	op_info->interval_ms_s, op_info->timeout_ms_s);
	lrmd_free_op_info(op_info);
	}
	g_list_free(op_list);

	} else if (options.api_call) {
	print_result("API-CALL FAILURE unknown action '%s'", options.action);
	test_exit(CRM_EX_ERROR);
	}

	if (rc < 0) {
	print_result("API-CALL FAILURE for '%s' api_rc:%d",
	options.api_call, rc);
	test_exit(CRM_EX_ERROR);
	}

	if (options.api_call && rc == pcmk_ok) {
	print_result("API-CALL SUCCESSFUL for '%s'", options.api_call);
	if (!options.listen) {
	test_exit(CRM_EX_OK);
	}
	}

	if (options.no_wait) {
	/* just make the call and exit regardless of anything else. */
	test_exit(CRM_EX_OK);
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Generate resource parameters from CIB if none explicitly given
	*
	* \return Standard Pacemaker return code
	*/
	static int
	generate_params(void)
	{
	int rc = pcmk_rc_ok;
	pcmk_scheduler_t *scheduler = NULL;
	xmlNode *cib_xml_copy = NULL;
	pcmk_resource_t *rsc = NULL;
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	if (options.params != NULL) {
	return pcmk_rc_ok; // User specified parameters explicitly
	}

	// Retrieve and update CIB
	rc = cib__signon_query(NULL, NULL, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	if (!cli_config_update(&cib_xml_copy, NULL, FALSE)) {
	crm_err("Could not update CIB");
	return pcmk_rc_cib_corrupt;
	}

	// Calculate cluster status
	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	crm_crit("Could not allocate scheduler data");
	return ENOMEM;
	}
	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	scheduler->input = cib_xml_copy;
	scheduler->now = crm_time_new(NULL);
	cluster_status(scheduler);

	// Find resource in CIB
	rsc = pe_find_resource_with_flags(scheduler->resources, options.rsc_id,
	pcmk_rsc_match_history
	\|pcmk_rsc_match_basename);
	if (rsc == NULL) {
	crm_err("Resource does not exist in config");
	pe_free_working_set(scheduler);
	return EINVAL;
	}

	// Add resource instance parameters to options.params
	params = pe_rsc_params(rsc, NULL, scheduler);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &value)) {
	options.params = lrmd_key_value_add(options.params, key, value);
	}
	}

	// Add resource meta-attributes to options.params
	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, NULL, scheduler);
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &value)) {
	char *crm_name = crm_meta_name(key);

	options.params = lrmd_key_value_add(options.params, crm_name, value);
	free(crm_name);
	}
	g_hash_table_destroy(meta);

	pe_free_working_set(scheduler);
	return rc;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	context = pcmk__build_arg_context(args, NULL, group, NULL);

	pcmk__add_main_args(context, basic_entries);
	pcmk__add_arg_group(context, "api-call", "API Call Options:",
	"Parameters for api-call option", api_call_entries);

	return context;
	}

	int
	main(int argc, char **argv)
	{
	GError *error = NULL;
	crm_exit_t exit_code = CRM_EX_OK;
	crm_trigger_t *trig = NULL;

	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	/* Typically we'd pass all the single character options that take an argument
	* as the second parameter here (and there's a bunch of those in this tool).
	* However, we control how this program is called so we can just not call it
	* in a way where the preprocessing ever matters.
	*/
	gchar **processed_args = pcmk__cmdline_preproc(argv, NULL);
	GOptionContext *context = build_arg_context(args, NULL);

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	/* We have to use crm_log_init here to set up the logging because there's
	* different handling for daemons vs. command line programs, and
	* pcmk__cli_init_logging is set up to only handle the latter.
	*/
	crm_log_init(NULL, LOG_INFO, TRUE, (args->verbosity? TRUE : FALSE), argc,
	argv, FALSE);

	for (int i = 0; i < args->verbosity; i++) {
	crm_bump_log_level(argc, argv);
	}

	if (!options.listen && pcmk__strcase_any_of(options.api_call, "metadata", "list_agents",
	"list_standards", "list_ocf_providers", NULL)) {
	options.no_connect = TRUE;
	}

	if (options.is_running) {
	int rc = pcmk_rc_ok;

	if (options.rsc_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"--is-running requires --rsc-id");
	goto done;
	}

	options.interval_ms = 0;
	if (options.timeout == 0) {
	options.timeout = 30000;
	}

	rc = generate_params();
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Can not determine resource status: "
	"unable to get parameters from CIB");
	goto done;
	}
	options.api_call = "exec";
	options.action = PCMK_ACTION_MONITOR;
	options.exec_call_opts = lrmd_opt_notify_orig_only;
	}

	if (!options.api_call && !options.listen) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must specify at least one of --api-call, --listen, "
	"or --is-running");
	goto done;
	}

	if (options.use_tls) {
	lrmd_conn = lrmd_remote_api_new(NULL, "localhost", 0);
	} else {
	lrmd_conn = lrmd_api_new();
	}
	trig = mainloop_add_trigger(G_PRIORITY_HIGH, start_test, NULL);
	mainloop_set_trigger(trig);
	mainloop_add_signal(SIGTERM, test_shutdown);

	crm_info("Starting");
	mainloop = g_main_loop_new(NULL, FALSE);
	g_main_loop_run(mainloop);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	free(key);
	free(val);

	pcmk__output_and_clear_error(&error, NULL);
	return test_exit(exit_code);
	}
	diff --git a/include/crm/common/scheduler_internal.h b/include/crm/common/scheduler_internal.h
	index 57e59eb35b..fa6bec7ed9 100644
	--- a/include/crm/common/scheduler_internal.h
	+++ b/include/crm/common/scheduler_internal.h
	@@ -1,143 +1,169 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
	# define PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H

	#include <crm/common/action_relation_internal.h>
	#include <crm/common/bundles_internal.h>
	#include <crm/common/clone_internal.h>
	#include <crm/common/digests_internal.h>
	#include <crm/common/failcounts_internal.h>
	#include <crm/common/group_internal.h>
	#include <crm/common/location_internal.h>
	#include <crm/common/roles_internal.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/* Some warnings are too noisy when logged every time a give function is called
	* (for example, using a deprecated feature). As an alternative, we allow
	* warnings to be logged once per scheduler sequence (transition). Each of those
	* warnings needs a flag defined here.
	*/
	enum pcmk__sched_warnings {
	pcmk__wo_blind = (1 << 0),
	pcmk__wo_restart_type = (1 << 1),
	pcmk__wo_role_after = (1 << 2),
	pcmk__wo_poweroff = (1 << 3),
	pcmk__wo_require_all = (1 << 4),
	pcmk__wo_order_score = (1 << 5),
	pcmk__wo_neg_threshold = (1 << 6),
	pcmk__wo_remove_after = (1 << 7),
	pcmk__wo_ping_node = (1 << 8),
	pcmk__wo_order_inst = (1 << 9),
	pcmk__wo_coloc_inst = (1 << 10),
	pcmk__wo_group_order = (1 << 11),
	pcmk__wo_group_coloc = (1 << 12),
	pcmk__wo_upstart = (1 << 13),
	pcmk__wo_nagios = (1 << 14),
	pcmk__wo_set_ordering = (1 << 15),
	};

	enum pcmk__check_parameters {
	/* Clear fail count if parameters changed for un-expired start or monitor
	* last_failure.
	*/
	pcmk__check_last_failure,

	/* Clear fail count if parameters changed for start, monitor, promote, or
	* migrate_from actions for active resources.
	*/
	pcmk__check_active,
	};

	// Group of enum pcmk__sched_warnings flags for warnings we want to log once
	extern uint32_t pcmk__warnings;

	/*!
	* \internal
	* \brief Log a resource-tagged message at info severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_info(rsc, fmt, args...) \
	crm_log_tag(LOG_INFO, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log a resource-tagged message at debug severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_debug(rsc, fmt, args...) \
	crm_log_tag(LOG_DEBUG, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log a resource-tagged message at trace severity
	*
	* \param[in] rsc Tag message with this resource's ID
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__rsc_trace(rsc, fmt, args...) \
	crm_log_tag(LOG_TRACE, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)

	/*!
	* \internal
	* \brief Log an error and remember that current scheduler input has errors
	*
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__sched_err(fmt...) do { \
	was_processing_error = TRUE; \
	crm_err(fmt); \
	} while (0)

	/*!
	* \internal
	* \brief Log a warning and remember that current scheduler input has warnings
	*
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__sched_warn(fmt...) do { \
	was_processing_warning = TRUE; \
	crm_warn(fmt); \
	} while (0)

	/*!
	* \internal
	* \brief Log a warning once per scheduler run
	*
	* \param[in] wo_flag enum pcmk__sched_warnings value for this warning
	* \param[in] fmt... printf(3)-style format and arguments
	*/
	#define pcmk__warn_once(wo_flag, fmt...) do { \
	if (!pcmk_is_set(pcmk__warnings, wo_flag)) { \
	if (wo_flag == pcmk__wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pcmk__config_warn(fmt); \
	} \
	pcmk__warnings = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Warn-once", "logging", \
	pcmk__warnings, \
	(wo_flag), #wo_flag); \
	} \
	} while (0)

	+/*!
	+ * \internal
	+ * \brief Set scheduler flags
	+ *
	+ * \param[in,out] scheduler Scheduler data
	+ * \param[in] flags_to_set Group of enum pcmk_scheduler_flags to set
	+ */
	+#define pcmk__set_scheduler_flags(scheduler, flags_to_set) do { \
	+ (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	+ LOG_TRACE, "Scheduler", crm_system_name, \
	+ (scheduler)->flags, (flags_to_set), #flags_to_set); \
	+ } while (0)
	+
	+/*!
	+ * \internal
	+ * \brief Clear scheduler flags
	+ *
	+ * \param[in,out] scheduler Scheduler data
	+ * \param[in] flags_to_clear Group of enum pcmk_scheduler_flags to clear
	+ */
	+#define pcmk__clear_scheduler_flags(scheduler, flags_to_clear) do { \
	+ (scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	+ LOG_TRACE, "Scheduler", crm_system_name, \
	+ (scheduler)->flags, (flags_to_clear), #flags_to_clear); \
	+ } while (0)
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index 1dfee80018..8e1c2d2abc 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,623 +1,611 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H

	# include <stdbool.h>
	# include <stdint.h>
	# include <string.h>
	# include <crm/msg_xml.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>
	# include <crm/common/scheduler_internal.h>

	const char pe__resource_description(const pcmk_resource_t rsc,
	uint32_t show_opts);

	bool pe__clone_is_ordered(const pcmk_resource_t *clone);
	int pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag);
	bool pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags);

	bool pe__group_flag_is_set(const pcmk_resource_t *group, uint32_t flags);
	pcmk_resource_t pe__last_group_member(const pcmk_resource_t group);

	-#define pe__set_working_set_flags(scheduler, flags_to_set) do { \
	- (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	- LOG_TRACE, "Scheduler", crm_system_name, \
	- (scheduler)->flags, (flags_to_set), #flags_to_set); \
	- } while (0)
	-
	-#define pe__clear_working_set_flags(scheduler, flags_to_clear) do { \
	- (scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	- LOG_TRACE, "Scheduler", crm_system_name, \
	- (scheduler)->flags, (flags_to_clear), #flags_to_clear); \
	- } while (0)
	-
	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	const pcmk_resource_t pe__const_top_resource(const pcmk_resource_t rsc,
	bool include_bundle);

	int pe__clone_max(const pcmk_resource_t *clone);
	int pe__clone_node_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_max(const pcmk_resource_t *clone);
	int pe__clone_promoted_node_max(const pcmk_resource_t *clone);
	void pe__create_clone_notifications(pcmk_resource_t *clone);
	void pe__free_clone_notification_data(pcmk_resource_t *clone);
	void pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
	pcmk_action_t *start,
	pcmk_action_t *started,
	pcmk_action_t *stop,
	pcmk_action_t *stopped);

	pcmk_action_t pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char *task,
	bool optional, bool runnable);

	void pe__create_promotable_pseudo_ops(pcmk_resource_t *clone,
	bool any_promoting, bool any_demoting);

	bool pe_can_fence(const pcmk_scheduler_t scheduler, const pcmk_node_t node);

	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pcmk_resource_t rsc, pcmk_node_t *node, gboolean create,
	const char name, pcmk_scheduler_t scheduler);
	pcmk_node_t native_location(const pcmk_resource_t rsc, GList **list,
	int current);

	void pe_metadata(pcmk__output_t *out);
	void verify_pe_options(GHashTable * options);

	void native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed);

	gboolean native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean group_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean clone_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);
	gboolean pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler);

	pcmk_resource_t native_find_rsc(pcmk_resource_t rsc, const char *id,
	const pcmk_node_t *node, int flags);

	gboolean native_active(pcmk_resource_t *rsc, gboolean all);
	gboolean group_active(pcmk_resource_t *rsc, gboolean all);
	gboolean clone_active(pcmk_resource_t *rsc, gboolean all);
	gboolean pe__bundle_active(pcmk_resource_t *rsc, gboolean all);

	//! \deprecated This function will be removed in a future release
	void native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void group_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void clone_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	//! \deprecated This function will be removed in a future release
	void pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data);

	gchar pcmk__native_output_string(const pcmk_resource_t rsc, const char *name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pcmk_node_t node, bool print_detail);


	// Clone notifications (pe_notif.c)
	void pe__order_notifs_after_fencing(const pcmk_action_t *action,
	pcmk_resource_t *rsc,
	pcmk_action_t *stonith_op);


	static inline const char *
	pe__rsc_bool_str(const pcmk_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_default(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_default(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);

	void native_free(pcmk_resource_t *rsc);
	void group_free(pcmk_resource_t *rsc);
	void clone_free(pcmk_resource_t *rsc);
	void pe__free_bundle(pcmk_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e group_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e clone_resource_state(const pcmk_resource_t *rsc,
	gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pcmk_resource_t *rsc,
	gboolean current);

	void pe__count_common(pcmk_resource_t *rsc);
	void pe__count_bundle(pcmk_resource_t *rsc);

	void common_free(pcmk_resource_t *rsc);

	pcmk_node_t pe__copy_node(const pcmk_node_t this_node);
	time_t get_effective_time(pcmk_scheduler_t *scheduler);

	/* Failure handling utilities (from failcounts.c) */

	int pe_get_failcount(const pcmk_node_t node, pcmk_resource_t rsc,
	time_t *last_failure, uint32_t flags,
	const xmlNode *xml_op);

	pcmk_action_t pe__clear_failcount(pcmk_resource_t rsc,
	const pcmk_node_t node, const char reason,
	pcmk_scheduler_t *scheduler);

	/* Functions for finding/counting a resource's active nodes */

	bool pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean);

	pcmk_node_t pe__find_active_requires(const pcmk_resource_t rsc,
	unsigned int *count);

	static inline pcmk_node_t *
	pe__current_node(const pcmk_resource_t *rsc)
	{
	return (rsc == NULL)? NULL : rsc->fns->active_node(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	GHashTable pe__node_list2table(const GList list);

	pcmk_action_t get_pseudo_op(const char name, pcmk_scheduler_t *scheduler);
	gboolean order_actions(pcmk_action_t lh_action, pcmk_action_t rh_action,
	uint32_t flags);

	void pe__show_node_scores_as(const char file, const char function,
	int line, bool to_log, const pcmk_resource_t *rsc,
	const char comment, GHashTable nodes,
	pcmk_scheduler_t *scheduler);

	#define pe__show_node_scores(level, rsc, text, nodes, scheduler) \
	pe__show_node_scores_as(__FILE__, __func__, __LINE__, \
	(level), (rsc), (text), (nodes), (scheduler))

	GHashTable pcmk__unpack_action_meta(pcmk_resource_t rsc,
	const pcmk_node_t *node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config);
	GHashTable pcmk__unpack_action_rsc_params(const xmlNode action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *data_set);
	xmlNode pcmk__find_action_config(const pcmk_resource_t rsc,
	const char *action_name, guint interval_ms,
	bool include_disabled);

	enum rsc_start_requirement pcmk__action_requires(const pcmk_resource_t *rsc,
	const char *action_name);

	enum action_fail_response pcmk__parse_on_fail(const pcmk_resource_t *rsc,
	const char *action_name,
	guint interval_ms,
	const char *value);

	enum rsc_role_e pcmk__role_after_failure(const pcmk_resource_t *rsc,
	const char *action_name,
	enum action_fail_response on_fail,
	GHashTable *meta);

	pcmk_action_t custom_action(pcmk_resource_t rsc, char key, const char task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler);

	# define delete_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), PCMK_ACTION_DELETE, node, \
	optional, rsc->cluster);

	# define stop_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), PCMK_ACTION_STOP, node, \
	optional, rsc->cluster);

	# define reload_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_RELOAD_AGENT, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), PCMK_ACTION_START, node, \
	optional, rsc->cluster)

	# define promote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), PCMK_ACTION_PROMOTE, node, \
	optional, rsc->cluster)

	# define demote_key(rsc) pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), PCMK_ACTION_DEMOTE, node, \
	optional, rsc->cluster)

	extern int pe_get_configured_timeout(pcmk_resource_t rsc, const char action,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t find_first_action(const GList input, const char *uuid,
	const char task, const pcmk_node_t on_node);

	enum action_tasks get_complex_task(const pcmk_resource_t *rsc,
	const char *name);

	GList find_actions(GList input, const char key, const pcmk_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pcmk_node_t *on_node);
	GList pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pcmk_action_t *action);

	void resource_location(pcmk_resource_t rsc, const pcmk_node_t node, int score,
	const char tag, pcmk_scheduler_t scheduler);

	extern int pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default);
	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	gboolean get_target_role(const pcmk_resource_t rsc, enum rsc_role_e role);
	void pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role,
	const char *why);

	pcmk_resource_t find_clone_instance(const pcmk_resource_t rsc,
	const char *sub_id);

	extern void destroy_ticket(gpointer data);
	pcmk_ticket_t ticket_new(const char ticket_id, pcmk_scheduler_t *scheduler);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(const pcmk_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return false;
	}

	int pe__target_rc_from_xml(const xmlNode *xml_op);

	gint pe__cmp_node_name(gconstpointer a, gconstpointer b);
	bool is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any);

	pcmk__op_digest_t pe__calculate_digests(pcmk_resource_t rsc, const char *task,
	guint *interval_ms,
	const pcmk_node_t *node,
	const xmlNode *xml_op,
	GHashTable *overrides,
	bool calc_secure,
	pcmk_scheduler_t *scheduler);

	void pe__free_digests(gpointer ptr);

	pcmk__op_digest_t rsc_action_digest_cmp(pcmk_resource_t rsc,
	const xmlNode *xml_op,
	pcmk_node_t *node,
	pcmk_scheduler_t *scheduler);

	pcmk_action_t pe_fence_op(pcmk_node_t node, const char *op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler);
	void trigger_unfencing(pcmk_resource_t rsc, pcmk_node_t node,
	const char reason, pcmk_action_t dependency,
	pcmk_scheduler_t *scheduler);

	char pe__action2reason(const pcmk_action_t action, enum pe_action_flags flag);
	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite);
	void pe__add_action_expected_result(pcmk_action_t *action, int expected_result);

	void pe__set_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pcmk_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_on_all(pcmk_scheduler_t *scheduler,
	uint64_t flag);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	//! \deprecated This function will be removed in a future release
	void print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, unsigned int options);
	void pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay);

	pcmk_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pcmk_scheduler_t scheduler);

	//! \deprecated This function will be removed in a future release
	void common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);
	int pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	unsigned int options);

	GList pe__bundle_containers(const pcmk_resource_t bundle);

	int pe__bundle_max(const pcmk_resource_t *rsc);
	bool pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node);
	pcmk_resource_t pe__bundled_resource(const pcmk_resource_t rsc);
	const pcmk_resource_t pe__get_rsc_in_container(const pcmk_resource_t instance);
	pcmk_resource_t pe__first_container(const pcmk_resource_t bundle);
	void pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data);
	void pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data);
	pcmk_resource_t pe__find_bundle_replica(const pcmk_resource_t bundle,
	const pcmk_node_t *node);
	bool pe__bundle_needs_remote_name(pcmk_resource_t *rsc);
	const char pe__add_bundle_remote_name(pcmk_resource_t rsc,
	pcmk_scheduler_t *scheduler,
	xmlNode xml, const char field);

	const char pe__node_attribute_calculated(const pcmk_node_t node,
	const char *name,
	const pcmk_resource_t *rsc,
	enum pcmk__rsc_node node_type,
	bool force_host);
	const char pe_node_attribute_raw(const pcmk_node_t node, const char *name);
	bool pe__is_universal_clone(const pcmk_resource_t *rsc,
	const pcmk_scheduler_t *scheduler);
	void pe__add_param_check(const xmlNode rsc_op, pcmk_resource_t rsc,
	pcmk_node_t *node, enum pcmk__check_parameters,
	pcmk_scheduler_t *scheduler);
	void pe__foreach_param_check(pcmk_scheduler_t *scheduler,
	void (cb)(pcmk_resource_t, pcmk_node_t*,
	const xmlNode*,
	enum pcmk__check_parameters));
	void pe__free_param_checks(pcmk_scheduler_t *scheduler);

	bool pe__shutdown_requested(const pcmk_node_t *node);
	void pe__update_recheck_time(time_t recheck, pcmk_scheduler_t *scheduler,
	const char *reason);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*
	* \param[in,out] out Output object to register messages with
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(const xmlNode xml_obj, const char set_name,
	const pe_rule_eval_data_t *rule_data,
	GHashTable hash, const char always_first,
	gboolean overwrite,
	pcmk_scheduler_t *scheduler);

	bool pe__resource_is_disabled(const pcmk_resource_t *rsc);
	void pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node);

	GList pe__rscs_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	GList pe__unames_with_tag(pcmk_scheduler_t scheduler, const char *tag_name);
	bool pe__rsc_has_tag(pcmk_scheduler_t scheduler, const char rsc,
	const char *tag);
	bool pe__uname_has_tag(pcmk_scheduler_t scheduler, const char node,
	const char *tag);

	bool pe__rsc_running_on_only(const pcmk_resource_t *rsc,
	const pcmk_node_t *node);
	bool pe__rsc_running_on_any(pcmk_resource_t rsc, GList node_list);
	GList pe__filter_rsc_list(GList rscs, GList *filter);
	GList * pe__build_node_name_list(pcmk_scheduler_t scheduler, const char s);
	GList * pe__build_rsc_list(pcmk_scheduler_t scheduler, const char s);

	bool pcmk__rsc_filtered_by_node(pcmk_resource_t rsc, GList only_node);

	gboolean pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__clone_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__group_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);
	gboolean pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent);

	xmlNode pe__failed_probe_for_rsc(const pcmk_resource_t rsc, const char *name);

	const char pe__clone_child_id(const pcmk_resource_t rsc);

	int pe__sum_node_health_scores(const pcmk_node_t *node, int base_health);
	int pe__node_health(pcmk_node_t *node);

	static inline enum pcmk__health_strategy
	pe__health_strategy(pcmk_scheduler_t *scheduler)
	{
	return pcmk__parse_health_strategy(pe_pref(scheduler->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	static inline int
	pe__health_score(const char option, pcmk_scheduler_t scheduler)
	{
	return char2score(pe_pref(scheduler->config_hash, option));
	}

	/*!
	* \internal
	* \brief Return a string suitable for logging as a node name
	*
	* \param[in] node Node to return a node name string for
	*
	* \return Node name if available, otherwise node ID if available,
	* otherwise "unspecified node" if node is NULL or "unidentified node"
	* if node has neither a name nor ID.
	*/
	static inline const char *
	pe__node_name(const pcmk_node_t *node)
	{
	if (node == NULL) {
	return "unspecified node";

	} else if (node->details->uname != NULL) {
	return node->details->uname;

	} else if (node->details->id != NULL) {
	return node->details->id;

	} else {
	return "unidentified node";
	}
	}

	/*!
	* \internal
	* \brief Check whether two node objects refer to the same node
	*
	* \param[in] node1 First node object to compare
	* \param[in] node2 Second node object to compare
	*
	* \return true if \p node1 and \p node2 refer to the same node
	*/
	static inline bool
	pe__same_node(const pcmk_node_t node1, const pcmk_node_t node2)
	{
	return (node1 != NULL) && (node2 != NULL)
	&& (node1->details == node2->details);
	}

	/*!
	* \internal
	* \brief Get the operation key from an action history entry
	*
	* \param[in] xml Action history entry
	*
	* \return Entry's operation key
	*/
	static inline const char *
	pe__xe_history_key(const xmlNode *xml)
	{
	if (xml == NULL) {
	return NULL;
	} else {
	/* @COMPAT Pacemaker <= 1.1.5 did not add the key, and used the ID
	* instead. Checking for that allows us to process old saved CIBs,
	* including some regression tests.
	*/
	const char *key = crm_element_value(xml, XML_LRM_ATTR_TASK_KEY);

	return pcmk__str_empty(key)? ID(xml) : key;
	}
	}

	#endif
	diff --git a/lib/pacemaker/pcmk_rule.c b/lib/pacemaker/pcmk_rule.c
	index 68b1b74724..0c673f8a05 100644
	--- a/lib/pacemaker/pcmk_rule.c
	+++ b/lib/pacemaker/pcmk_rule.c
	@@ -1,296 +1,296 @@
	/*
	* Copyright 2022-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/cib/internal.h>
	#include <crm/common/cib.h>
	#include <crm/common/iso8601.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/rules_internal.h>
	#include <pacemaker-internal.h>

	/*!
	* \internal
	* \brief Evaluate a date expression for a specific time
	*
	* \param[in] expr date_expression XML
	* \param[in] now Time for which to evaluate expression
	*
	* \return Standard Pacemaker return code
	*/
	static int
	eval_date_expression(const xmlNode expr, crm_time_t now)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe__eval_date_expr(expr, &rule_data, NULL);
	}

	/*!
	* \internal
	* \brief Initialize scheduler data for checking rules
	*
	* Make our own copies of the CIB XML and date/time object, if they're not
	* \c NULL. This way we don't have to take ownership of the objects passed via
	* the API.
	*
	* \param[in,out] out Output object
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Check whether the rule is in effect at this date
	* and time (if \c NULL, use current date and time)
	* \param[out] scheduler Where to store initialized scheduler data
	*
	* \return Standard Pacemaker return code
	*/
	static int
	init_rule_check(pcmk__output_t out, xmlNodePtr input, const crm_time_t date,
	pcmk_scheduler_t **scheduler)
	{
	// Allows for cleaner syntax than dereferencing the scheduler argument
	pcmk_scheduler_t *new_scheduler = NULL;

	new_scheduler = pe_new_working_set();
	if (new_scheduler == NULL) {
	return ENOMEM;
	}

	- pe__set_working_set_flags(new_scheduler,
	+ pcmk__set_scheduler_flags(new_scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);

	// Populate the scheduler data

	// Make our own copy of the given input or fetch the CIB and use that
	if (input != NULL) {
	new_scheduler->input = copy_xml(input);
	if (new_scheduler->input == NULL) {
	out->err(out, "Failed to copy input XML");
	pe_free_working_set(new_scheduler);
	return ENOMEM;
	}

	} else {
	int rc = cib__signon_query(out, NULL, &(new_scheduler->input));

	if (rc != pcmk_rc_ok) {
	pe_free_working_set(new_scheduler);
	return rc;
	}
	}

	// Make our own copy of the given crm_time_t object; otherwise
	// cluster_status() populates with the current time
	if (date != NULL) {
	// pcmk_copy_time() guarantees non-NULL
	new_scheduler->now = pcmk_copy_time(date);
	}

	// Unpack everything
	cluster_status(new_scheduler);
	*scheduler = new_scheduler;

	return pcmk_rc_ok;
	}

	#define XPATH_NODE_RULE "//" XML_TAG_RULE "[@" PCMK_XA_ID "='%s']"

	/*!
	* \internal
	* \brief Check whether a given rule is in effect
	*
	* \param[in] scheduler Scheduler data
	* \param[in] rule_id The ID of the rule to check
	* \param[out] error Where to store a rule evaluation error message
	*
	* \return Standard Pacemaker return code
	*/
	static int
	eval_rule(pcmk_scheduler_t scheduler, const char rule_id, const char **error)
	{
	xmlNodePtr cib_constraints = NULL;
	xmlNodePtr match = NULL;
	xmlXPathObjectPtr xpath_obj = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;
	int num_results = 0;

	*error = NULL;

	/* Rules are under the constraints node in the XML, so first find that. */
	cib_constraints = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS);

	/* Get all rules matching the given ID that are also simple enough for us
	* to check. For the moment, these rules must only have a single
	* date_expression child and:
	* - Do not have a date_spec operation, or
	* - Have a date_spec operation that contains years= but does not contain
	* moon=.
	*
	* We do this in steps to provide better error messages. First, check that
	* there's any rule with the given ID.
	*/
	xpath = crm_strdup_printf(XPATH_NODE_RULE, rule_id);
	xpath_obj = xpath_search(cib_constraints, xpath);
	num_results = numXpathResults(xpath_obj);

	free(xpath);
	freeXpathObject(xpath_obj);

	if (num_results == 0) {
	*error = "Rule not found";
	return ENXIO;
	}

	if (num_results > 1) {
	// Should not be possible; schema prevents this
	*error = "Found more than one rule with matching ID";
	return pcmk_rc_duplicate_id;
	}

	/* Next, make sure it has exactly one date_expression. */
	xpath = crm_strdup_printf(XPATH_NODE_RULE "//date_expression", rule_id);
	xpath_obj = xpath_search(cib_constraints, xpath);
	num_results = numXpathResults(xpath_obj);

	free(xpath);
	freeXpathObject(xpath_obj);

	if (num_results != 1) {
	if (num_results == 0) {
	*error = "Rule does not have a date expression";
	} else {
	*error = "Rule has more than one date expression";
	}
	return EOPNOTSUPP;
	}

	/* Then, check that it's something we actually support. */
	xpath = crm_strdup_printf(XPATH_NODE_RULE "//date_expression["
	"@" XML_EXPR_ATTR_OPERATION "!='date_spec']",
	rule_id);
	xpath_obj = xpath_search(cib_constraints, xpath);
	num_results = numXpathResults(xpath_obj);

	free(xpath);

	if (num_results == 0) {
	freeXpathObject(xpath_obj);

	xpath = crm_strdup_printf(XPATH_NODE_RULE "//date_expression["
	"@" XML_EXPR_ATTR_OPERATION "='date_spec' "
	"and date_spec/@years "
	"and not(date_spec/@moon)]", rule_id);
	xpath_obj = xpath_search(cib_constraints, xpath);
	num_results = numXpathResults(xpath_obj);

	free(xpath);

	if (num_results == 0) {
	freeXpathObject(xpath_obj);
	*error = "Rule must either not use date_spec, or use date_spec "
	"with years= but not moon=";
	return EOPNOTSUPP;
	}
	}

	match = getXpathResult(xpath_obj, 0);

	/* We should have ensured this with the xpath query above, but double-
	* checking can't hurt.
	*/
	CRM_ASSERT(match != NULL);
	CRM_ASSERT(find_expression_type(match) == pcmk__subexpr_datetime);

	rc = eval_date_expression(match, scheduler->now);
	if (rc == pcmk_rc_undetermined) {
	/* pe__eval_date_expr() should return this only if something is
	* malformed or missing
	*/
	*error = "Error parsing rule";
	}

	freeXpathObject(xpath_obj);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether each rule in a list is in effect
	*
	* \param[in,out] out Output object
	* \param[in] input The CIB XML to check (if \c NULL, use current CIB)
	* \param[in] date Check whether the rule is in effect at this date and
	* time (if \c NULL, use current date and time)
	* \param[in] rule_ids The IDs of the rules to check, as a <tt>NULL</tt>-
	* terminated list.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__check_rules(pcmk__output_t out, xmlNodePtr input, const crm_time_t date,
	const char **rule_ids)
	{
	pcmk_scheduler_t *scheduler = NULL;
	int rc = pcmk_rc_ok;

	CRM_ASSERT(out != NULL);

	if (rule_ids == NULL) {
	// Trivial case; every rule specified is in effect
	return pcmk_rc_ok;
	}

	rc = init_rule_check(out, input, date, &scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	for (const char *rule_id = rule_ids; rule_id != NULL; rule_id++) {
	const char *error = NULL;
	int last_rc = eval_rule(scheduler, *rule_id, &error);

	out->message(out, "rule-check", *rule_id, last_rc, error);

	if (last_rc != pcmk_rc_ok) {
	rc = last_rc;
	}
	}

	pe_free_working_set(scheduler);
	return rc;
	}

	// Documented in pacemaker.h
	int
	pcmk_check_rules(xmlNodePtr xml, xmlNodePtr input, const crm_time_t date,
	const char **rule_ids)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pcmk__register_lib_messages(out);

	rc = pcmk__check_rules(out, input, date, rule_ids);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
	index 39171d2789..d9e0162dde 100644
	--- a/lib/pacemaker/pcmk_scheduler.c
	+++ b/lib/pacemaker/pcmk_scheduler.c
	@@ -1,821 +1,821 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	CRM_TRACE_INIT_DATA(pacemaker);

	/*!
	* \internal
	* \brief Do deferred action checks after assignment
	*
	* When unpacking the resource history, the scheduler checks for resource
	* configurations that have changed since an action was run. However, at that
	* time, bundles using the REMOTE_CONTAINER_HACK don't have their final
	* parameter information, so instead they add a deferred check to a list. This
	* function processes one entry in that list.
	*
	* \param[in,out] rsc Resource that action history is for
	* \param[in,out] node Node that action history is for
	* \param[in] rsc_op Action history entry
	* \param[in] check Type of deferred check to do
	*/
	static void
	check_params(pcmk_resource_t rsc, pcmk_node_t node, const xmlNode *rsc_op,
	enum pcmk__check_parameters check)
	{
	const char *reason = NULL;
	pcmk__op_digest_t *digest_data = NULL;

	switch (check) {
	case pcmk__check_active:
	if (pcmk__check_action_config(rsc, node, rsc_op)
	&& pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL)) {
	reason = "action definition changed";
	}
	break;

	case pcmk__check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s has "
	"no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}
	if (reason != NULL) {
	pe__clear_failcount(rsc, node, reason, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has failcount clearing scheduled on a node
	*
	* \param[in] node Node to check
	* \param[in] rsc Resource to check
	*
	* \return true if \p rsc has failcount clearing scheduled on \p node,
	* otherwise false
	*/
	static bool
	failcount_clear_action_exists(const pcmk_node_t *node,
	const pcmk_resource_t *rsc)
	{
	GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
	TRUE);

	if (list != NULL) {
	g_list_free(list);
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it reached its failure threshold there
	*
	* \param[in,out] data Resource to check failure threshold for
	* \param[in] user_data Node to check resource on
	*/
	static void
	check_failure_threshold(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, check_failure_threshold, user_data);
	return;
	}

	if (!failcount_clear_action_exists(node, rsc)) {
	/* Don't force the resource away from this node due to a failcount
	* that's going to be cleared.
	*
	* @TODO Failcount clearing can be scheduled in
	* pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
	* schedule_resource_actions() via check_params(). This runs well before
	* then, so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't. Worst case, we stop or move the
	* resource, then move it back in the next transition.
	*/
	pcmk_resource_t *failed = NULL;

	if (pcmk__threshold_reached(rsc, node, &failed)) {
	resource_location(failed, node, -INFINITY, "__fail_limit__",
	rsc->cluster);
	}
	}
	}

	/*!
	* \internal
	* \brief If resource has exclusive discovery, ban node if not allowed
	*
	* Location constraints have a resource-discovery option that allows users to
	* specify where probes are done for the affected resource. If this is set to
	* exclusive, probes will only be done on nodes listed in exclusive constraints.
	* This function bans the resource from the node if the node is not listed.
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Node to check resource on
	*/
	static void
	apply_exclusive_discovery(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;

	if (rsc->exclusive_discover
	\|\| pe__const_top_resource(rsc, false)->exclusive_discover) {
	pcmk_node_t *match = NULL;

	// If this is a collective resource, apply recursively to children
	g_list_foreach(rsc->children, apply_exclusive_discovery, user_data);

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if ((match != NULL)
	&& (match->rsc_discover_mode != pcmk_probe_exclusive)) {
	match->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Apply stickiness to a resource if appropriate
	*
	* \param[in,out] data Resource to check for stickiness
	* \param[in] user_data Ignored
	*/
	static void
	apply_stickiness(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	pcmk_node_t *node = NULL;

	// If this is a collective resource, apply recursively to children instead
	if (rsc->children != NULL) {
	g_list_foreach(rsc->children, apply_stickiness, NULL);
	return;
	}

	/* A resource is sticky if it is managed, has stickiness configured, and is
	* active on a single node.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| (rsc->stickiness < 1) \|\| !pcmk__list_of_1(rsc->running_on)) {
	return;
	}

	node = rsc->running_on->data;

	/* In a symmetric cluster, stickiness can always be used. In an
	* asymmetric cluster, we have to check whether the resource is still
	* allowed on the node, so we don't keep the resource somewhere it is no
	* longer explicitly enabled.
	*/
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (g_hash_table_lookup(rsc->allowed_nodes,
	node->details->id) == NULL)) {
	pcmk__rsc_debug(rsc,
	"Ignoring %s stickiness because the cluster is "
	"asymmetric and %s is not explicitly allowed",
	rsc->id, pe__node_name(node));
	return;
	}

	pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
	rsc->id, rsc->stickiness, pe__node_name(node));
	resource_location(rsc, node, rsc->stickiness, "stickiness", rsc->cluster);
	}

	/*!
	* \internal
	* \brief Apply shutdown locks for all resources as appropriate
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	apply_shutdown_locks(pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	return;
	}
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->shutdown_lock(rsc);
	}
	}

	/*!
	* \internal
	* \brief Calculate the number of available nodes in the cluster
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	count_available_nodes(pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(scheduler->flags, pcmk_sched_no_compat)) {
	return;
	}

	// @COMPAT for API backward compatibility only (cluster does not use value)
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if ((node != NULL) && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	scheduler->max_valid_nodes++;
	}
	}
	crm_trace("Online node count: %d", scheduler->max_valid_nodes);
	}

	/*
	* \internal
	* \brief Apply node-specific scheduling criteria
	*
	* After the CIB has been unpacked, process node-specific scheduling criteria
	* including shutdown locks, location constraints, resource stickiness,
	* migration thresholds, and exclusive resource discovery.
	*/
	static void
	apply_node_criteria(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Applying node-specific scheduling criteria");
	apply_shutdown_locks(scheduler);
	count_available_nodes(scheduler);
	pcmk__apply_locations(scheduler);
	g_list_foreach(scheduler->resources, apply_stickiness, NULL);

	for (GList *node_iter = scheduler->nodes; node_iter != NULL;
	node_iter = node_iter->next) {
	for (GList *rsc_iter = scheduler->resources; rsc_iter != NULL;
	rsc_iter = rsc_iter->next) {
	check_failure_threshold(rsc_iter->data, node_iter->data);
	apply_exclusive_discovery(rsc_iter->data, node_iter->data);
	}
	}
	}

	/*!
	* \internal
	* \brief Assign resources to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	assign_resources(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;

	crm_trace("Assigning resources to nodes");

	if (!pcmk__str_eq(scheduler->placement_strategy, "default",
	pcmk__str_casei)) {
	pcmk__sort_resources(scheduler);
	}
	pcmk__show_node_capacities("Original", scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	/* Assign remote connection resources first (which will also assign any
	* colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
	rsc->id);
	rsc->cmds->assign(rsc, rsc->partial_migration_target, true);
	}
	}
	}

	/* now do the rest of the resources */
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (!rsc->is_remote_node) {
	pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
	rsc->xml->name, rsc->id);
	rsc->cmds->assign(rsc, NULL, true);
	}
	}

	pcmk__show_node_capacities("Remaining", scheduler);
	}

	/*!
	* \internal
	* \brief Schedule fail count clearing on online nodes if resource is orphaned
	*
	* \param[in,out] data Resource to check
	* \param[in] user_data Ignored
	*/
	static void
	clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	return;
	}
	crm_trace("Clear fail counts for orphaned resource %s", rsc->id);

	/* There's no need to recurse into rsc->children because those
	* should just be unassigned clone instances.
	*/

	for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *clear_op = NULL;

	if (!node->details->online) {
	continue;
	}
	if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
	continue;
	}

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	rsc->cluster);

	/* We can't use order_action_then_stop() here because its
	* pcmk__ar_guest_allowed breaks things
	*/
	pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
	NULL, pcmk__ar_ordered, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Schedule any resource actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_resource_actions(pcmk_scheduler_t *scheduler)
	{
	// Process deferred action checks
	pe__foreach_param_check(scheduler, check_params);
	pe__free_param_checks(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_trace("Scheduling probes");
	pcmk__schedule_probes(scheduler);
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	g_list_foreach(scheduler->resources, clear_failcounts_if_orphaned,
	NULL);
	}

	crm_trace("Scheduling resource actions");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->create_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource or any of its descendants are managed
	*
	* \param[in] rsc Resource to check
	*
	* \return true if resource or any descendant is managed, otherwise false
	*/
	static bool
	is_managed(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return true;
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	if (is_managed((pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether any resources in the cluster are managed
	*
	* \param[in] scheduler Scheduler data
	*
	* \return true if any resource is managed, otherwise false
	*/
	static bool
	any_managed_resources(const pcmk_scheduler_t *scheduler)
	{
	for (const GList *iter = scheduler->resources;
	iter != NULL; iter = iter->next) {
	if (is_managed((const pcmk_resource_t *) iter->data)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a node requires fencing
	*
	* \param[in] node Node to check
	* \param[in] have_managed Whether any resource in cluster is managed
	*
	* \return true if \p node should be fenced, otherwise false
	*/
	static bool
	needs_fencing(const pcmk_node_t *node, bool have_managed)
	{
	return have_managed && node->details->unclean
	&& pe_can_fence(node->details->data_set, node);
	}

	/*!
	* \internal
	* \brief Check whether a node requires shutdown
	*
	* \param[in] node Node to check
	*
	* \return true if \p node should be shut down, otherwise false
	*/
	static bool
	needs_shutdown(const pcmk_node_t *node)
	{
	if (pe__is_guest_or_remote_node(node)) {
	/* Do not send shutdown actions for Pacemaker Remote nodes.
	* @TODO We might come up with a good use for this in the future.
	*/
	return false;
	}
	return node->details->online && node->details->shutdown;
	}

	/*!
	* \internal
	* \brief Track and order non-DC fencing
	*
	* \param[in,out] list List of existing non-DC fencing actions
	* \param[in,out] action Fencing action to prepend to \p list
	* \param[in] scheduler Scheduler data
	*
	* \return (Possibly new) head of \p list
	*/
	static GList *
	add_nondc_fencing(GList list, pcmk_action_t action,
	const pcmk_scheduler_t *scheduler)
	{
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)
	&& (list != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
	}
	return g_list_prepend(list, action);
	}

	/*!
	* \internal
	* \brief Schedule a node for fencing
	*
	* \param[in,out] node Node that requires fencing
	*/
	static pcmk_action_t *
	schedule_fencing(pcmk_node_t *node)
	{
	pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
	FALSE, node->details->data_set);

	pcmk__sched_warn("Scheduling node %s for fencing", pe__node_name(node));
	pcmk__order_vs_fence(fencing, node->details->data_set);
	return fencing;
	}

	/*!
	* \internal
	* \brief Create and order node fencing and shutdown actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *dc_down = NULL;
	bool integrity_lost = false;
	bool have_managed = any_managed_resources(scheduler);
	GList *fencing_ops = NULL;
	GList *shutdown_ops = NULL;

	crm_trace("Scheduling fencing and shutdowns as needed");
	if (!have_managed) {
	crm_notice("No fencing will be done until there are resources "
	"to manage");
	}

	// Check each node for whether it needs fencing or shutdown
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	pcmk_action_t *fencing = NULL;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && have_managed
	&& pe_can_fence(scheduler, node)) {
	pcmk__fence_guest(node);
	}
	continue;
	}

	if (needs_fencing(node, have_managed)) {
	fencing = schedule_fencing(node);

	// Track DC and non-DC fence actions separately
	if (node->details->is_dc) {
	dc_down = fencing;
	} else {
	fencing_ops = add_nondc_fencing(fencing_ops, fencing,
	scheduler);
	}

	} else if (needs_shutdown(node)) {
	pcmk_action_t *down_op = pcmk__new_shutdown_action(node);

	// Track DC and non-DC shutdown actions separately
	if (node->details->is_dc) {
	dc_down = down_op;
	} else {
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if ((fencing == NULL) && node->details->unclean) {
	integrity_lost = true;
	pcmk__config_warn("Node %s is unclean but cannot be fenced",
	pe__node_name(node));
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resource functionality and data integrity "
	"cannot be guaranteed (configure, enable, "
	"and test fencing to correct this)");

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	crm_notice("Unclean nodes will not be fenced until quorum is "
	"attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
	"ignore");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	pcmk__order_after_each(dc_down, shutdown_ops);
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	pcmk__order_after_each(dc_down, fencing_ops);
	} else if (fencing_ops != NULL) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
	pcmk__ar_ordered);
	}
	}
	g_list_free(fencing_ops);
	g_list_free(shutdown_ops);
	}

	static void
	log_resource_details(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;

	/* Due to the `crm_mon --node=` feature, out->message() for all the
	* resource-related messages expects a list of nodes that we are allowed to
	* output information for. Here, we create a wildcard to match all nodes.
	*/
	all = g_list_prepend(all, (gpointer) "*");

	for (GList *item = scheduler->resources; item != NULL; item = item->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) item->data;

	// Log all resources except inactive orphans
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role != pcmk_role_stopped)) {
	out->message(out, crm_map_element_name(rsc->xml), 0, rsc, all, all);
	}
	}

	g_list_free(all);
	}

	static void
	log_all_actions(pcmk_scheduler_t *scheduler)
	{
	/* This only ever outputs to the log, so ignore whatever output object was
	* previously set and just log instead.
	*/
	pcmk__output_t *prev_out = scheduler->priv;
	pcmk__output_t *out = NULL;

	if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
	return;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);
	pcmk__output_set_log_level(out, LOG_NOTICE);
	scheduler->priv = out;

	out->begin_list(out, NULL, NULL, "Actions");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);

	scheduler->priv = prev_out;
	}

	/*!
	* \internal
	* \brief Log all required but unrunnable actions at trace level
	*
	* \param[in] scheduler Scheduler data
	*/
	static void
	log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
	{
	const uint64_t flags = pcmk_action_optional
	\|pcmk_action_runnable
	\|pcmk_action_pseudo;

	crm_trace("Required but unrunnable actions:");
	for (const GList *iter = scheduler->actions;
	iter != NULL; iter = iter->next) {

	const pcmk_action_t action = (const pcmk_action_t ) iter->data;

	if (!pcmk_any_flags_set(action->flags, flags)) {
	pcmk__log_action("\t", action, true);
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack the CIB for scheduling
	*
	* \param[in,out] cib CIB XML to unpack (may be NULL if already unpacked)
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_cib(xmlNode cib, unsigned long long flags, pcmk_scheduler_t scheduler)
	{
	const char* localhost_save = NULL;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_have_status)) {
	crm_trace("Reusing previously calculated cluster status");
	- pe__set_working_set_flags(scheduler, flags);
	+ pcmk__set_scheduler_flags(scheduler, flags);
	return;
	}

	if (scheduler->localhost) {
	localhost_save = scheduler->localhost;
	}

	CRM_ASSERT(cib != NULL);
	crm_trace("Calculating cluster status");

	/* This will zero the entire struct without freeing anything first, so
	* callers should never call pcmk__schedule_actions() with a populated data
	* set unless pcmk_sched_have_status is set (i.e. cluster_status() was
	* previously called, whether directly or via pcmk__schedule_actions()).
	*/
	set_working_set_defaults(scheduler);

	if (localhost_save) {
	scheduler->localhost = localhost_save;
	}

	- pe__set_working_set_flags(scheduler, flags);
	+ pcmk__set_scheduler_flags(scheduler, flags);
	scheduler->input = cib;
	cluster_status(scheduler); // Sets pcmk_sched_have_status
	}

	/*!
	* \internal
	* \brief Run the scheduler for a given CIB
	*
	* \param[in,out] cib CIB XML to use as scheduler input
	* \param[in] flags Scheduler flags to set in addition to defaults
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
	pcmk_scheduler_t *scheduler)
	{
	unpack_cib(cib, flags, scheduler);
	pcmk__set_assignment_methods(scheduler);
	pcmk__apply_node_health(scheduler);
	pcmk__unpack_constraints(scheduler);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_validate_only)) {
	return;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)
	&& pcmk__is_daemon) {
	log_resource_details(scheduler);
	}

	apply_node_criteria(scheduler);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	return;
	}

	pcmk__create_internal_constraints(scheduler);
	pcmk__handle_rsc_config_changes(scheduler);
	assign_resources(scheduler);
	schedule_resource_actions(scheduler);

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we can mark nodes as needing fencing.
	*/
	pcmk__order_remote_connection_actions(scheduler);

	schedule_fencing_and_shutdowns(scheduler);
	pcmk__apply_orderings(scheduler);
	log_all_actions(scheduler);
	pcmk__create_graph(scheduler);

	if (get_crm_log_level() == LOG_TRACE) {
	log_unrunnable_actions(scheduler);
	}
	}
	diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
	index 1d3623ccf6..999f5eb167 100644
	--- a/lib/pacemaker/pcmk_simulate.c
	+++ b/lib/pacemaker/pcmk_simulate.c
	@@ -1,1006 +1,1006 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/scheduler.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <stdint.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	static pcmk__output_t *out = NULL;
	static cib_t *fake_cib = NULL;
	static GList *fake_resource_list = NULL;
	static const GList *fake_op_fail_list = NULL;

	static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date);

	/*!
	* \internal
	* \brief Create an action name for use in a dot graph
	*
	* \param[in] action Action to create name for
	* \param[in] verbose If true, add action ID to name
	*
	* \return Newly allocated string with action name
	* \note It is the caller's responsibility to free the result.
	*/
	static char *
	create_action_name(const pcmk_action_t *action, bool verbose)
	{
	char *action_name = NULL;
	const char *prefix = "";
	const char *action_host = NULL;
	const char *clone_name = NULL;
	const char *task = action->task;

	if (action->node != NULL) {
	action_host = action->node->details->uname;
	} else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_host = "<none>";
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	prefix = "Cancel ";
	task = action->cancel_task;
	}

	if (action->rsc != NULL) {
	clone_name = action->rsc->clone_name;
	}

	if (clone_name != NULL) {
	char *key = NULL;
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	const char *n_type = g_hash_table_lookup(action->meta,
	"notify_key_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_key_operation");

	CRM_ASSERT(n_type != NULL);
	CRM_ASSERT(n_task != NULL);
	key = pcmk__notify_key(clone_name, n_type, n_task);
	} else {
	key = pcmk__op_key(clone_name, task, interval_ms);
	}

	if (action_host != NULL) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, key, action_host);
	} else {
	action_name = crm_strdup_printf("%s%s", prefix, key);
	}
	free(key);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	action_name = crm_strdup_printf("%s%s '%s' %s",
	prefix, action->task, op, action_host);

	} else if (action->rsc && action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->uuid, action_host);

	} else if (action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->task, action_host);

	} else {
	action_name = crm_strdup_printf("%s", action->uuid);
	}

	if (verbose) {
	char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);

	free(action_name);
	action_name = with_id;
	}
	return action_name;
	}

	/*!
	* \internal
	* \brief Display the status of a cluster
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] show_opts How to modify display (as pcmk_show_opt_e flags)
	* \param[in] section_opts Sections to display (as pcmk_section_e flags)
	* \param[in] title What to use as list title
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts,
	uint32_t section_opts, const char *title,
	bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;
	crm_exit_t stonith_rc = 0;
	enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;

	section_opts \|= pcmk_section_nodes \| pcmk_section_resources;
	show_opts \|= pcmk_show_inactive_rscs \| pcmk_show_failed_detail;

	all = g_list_prepend(all, (gpointer) "*");

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "%s", title);
	out->message(out, "cluster-status",
	scheduler, state, stonith_rc, NULL,
	false, section_opts, show_opts, NULL, all, all);
	out->end_list(out);

	g_list_free(all);
	}

	/*!
	* \internal
	* \brief Display a summary of all actions scheduled in a transition
	*
	* \param[in,out] scheduler Scheduler data (fully scheduled)
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "Transition Summary");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	}

	/*!
	* \internal
	* \brief Reset scheduler input, output, date, and flags
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] input What to set as cluster input
	* \param[in] out What to set as cluster output object
	* \param[in] use_date What to set as cluster's current timestamp
	* \param[in] flags Group of enum pcmk_scheduler_flags to set
	*/
	static void
	reset(pcmk_scheduler_t scheduler, xmlNodePtr input, pcmk__output_t out,
	const char *use_date, unsigned int flags)
	{
	scheduler->input = input;
	scheduler->priv = out;
	set_effective_date(scheduler, true, use_date);
	if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_sanitized);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_output_scores);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_show_utilization);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}
	}

	/*!
	* \brief Write out a file in dot(1) format describing the actions that will
	* be taken by the scheduler in response to an input CIB file.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] dot_file The filename to write
	* \param[in] all_actions Write all actions, even those that are optional
	* or are on unmanaged resources
	* \param[in] verbose Add extra information, such as action IDs, to the
	* output
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_sim_dotfile(pcmk_scheduler_t scheduler, const char dot_file,
	bool all_actions, bool verbose)
	{
	GList *iter = NULL;
	FILE *dot_strm = fopen(dot_file, "w");

	if (dot_strm == NULL) {
	return errno;
	}

	fprintf(dot_strm, " digraph \"g\" {\n");
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	const char *style = "dashed";
	const char *font = "black";
	const char *color = "black";
	char *action_name = create_action_name(action, verbose);

	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	font = "orange";
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) {
	style = "bold";
	color = "green";

	} else if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) {
	color = "red";
	font = "purple";
	if (!all_actions) {
	goto do_not_write;
	}

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	color = "blue";
	if (!all_actions) {
	goto do_not_write;
	}

	} else {
	color = "red";
	CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable));
	}

	pe__set_action_flags(action, pcmk_action_added_to_graph);
	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
	action_name, style, color, font);
	do_not_write:
	free(action_name);
	}

	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	for (GList *before_iter = action->actions_before;
	before_iter != NULL; before_iter = before_iter->next) {

	pcmk__related_action_t *before = before_iter->data;

	char *before_name = NULL;
	char *after_name = NULL;
	const char *style = "dashed";
	bool optional = true;

	if (before->state == pe_link_dumped) {
	optional = false;
	style = "bold";
	} else if ((uint32_t) before->type == pcmk__ar_none) {
	continue;
	} else if (pcmk_is_set(before->action->flags,
	pcmk_action_added_to_graph)
	&& pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	&& (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) {
	optional = false;
	}

	if (all_actions \|\| !optional) {
	before_name = create_action_name(before->action, verbose);
	after_name = create_action_name(action, verbose);
	fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
	before_name, after_name, style);
	free(before_name);
	free(after_name);
	}
	}
	}

	fprintf(dot_strm, "}\n");
	fflush(dot_strm);
	fclose(dot_strm);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Profile the configuration updates and scheduler actions in a single
	* CIB file, printing the profiling timings.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in] xml_file The CIB file to profile
	* \param[in] repeat Number of times to run
	* \param[in,out] scheduler Scheduler data
	* \param[in] use_date The date to set the cluster's time to (may be NULL)
	*/
	static void
	profile_file(const char *xml_file, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	xmlNode *cib_object = NULL;
	clock_t start = 0;
	clock_t end;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	CRM_ASSERT(out != NULL);

	cib_object = filename2xml(xml_file);
	start = clock();

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	for (int i = 0; i < repeat; ++i) {
	xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);

	scheduler->input = input;
	set_effective_date(scheduler, false, use_date);
	pcmk__schedule_actions(input, scheduler_flags, scheduler);
	pe_reset_working_set(scheduler);
	}

	end = clock();
	out->message(out, "profile", xml_file, start, end);
	}

	void
	pcmk__profile_dir(const char *dir, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	struct dirent **namelist;

	int file_num = scandir(dir, &namelist, 0, alphasort);

	CRM_ASSERT(out != NULL);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX];

	out->begin_list(out, NULL, NULL, "Timings");

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
	".xml")) {
	free(namelist[file_num]);
	continue;
	}
	snprintf(buffer, sizeof(buffer), "%s/%s",
	dir, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	profile_file(buffer, repeat, scheduler, use_date);
	}
	free(namelist[file_num]);
	}
	free(namelist);

	out->end_list(out);
	}
	}

	/*!
	* \brief Set the date of the cluster, either to the value given by
	* \p use_date, or to the "execution-date" value in the CIB.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] print_original If \p true, the "execution-date" should
	* also be printed
	* \param[in] use_date The date to set the cluster's time to
	* (may be NULL)
	*/
	static void
	set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	time_t original_date = 0;

	CRM_ASSERT(out != NULL);

	crm_element_value_epoch(scheduler->input, "execution-date", &original_date);

	if (use_date) {
	scheduler->now = crm_time_new(use_date);
	out->info(out, "Setting effective cluster time: %s", use_date);
	crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now,
	crm_time_log_date \| crm_time_log_timeofday);

	} else if (original_date != 0) {
	scheduler->now = pcmk__copy_timet(original_date);

	if (print_original) {
	char *when = crm_time_as_string(scheduler->now,
	crm_time_log_date\|crm_time_log_timeofday);

	out->info(out, "Using the original execution date of: %s", when);
	free(when);
	}
	}
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a pseudo-action in a graph
	*
	* \param[in,out] graph Graph to update with pseudo-action result
	* \param[in,out] action Pseudo-action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-pseudo-action", node, task);

	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate executing a resource action in a graph
	*
	* \param[in,out] graph Graph to update with resource action result
	* \param[in,out] action Resource action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_resource_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	int rc;
	lrmd_event_data_t *op = NULL;
	int target_outcome = PCMK_OCF_OK;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *resource = NULL;
	const char *rprovider = NULL;
	const char *resource_config_name = NULL;
	const char *operation = crm_element_value(action->xml, "operation");
	const char *target_rc_s = crm_meta_value(action->params,
	XML_ATTR_TE_TARGET_RC);

	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	char *uuid = NULL;
	const char *router_node = crm_element_value(action->xml,
	XML_LRM_ATTR_ROUTER_NODE);

	// Certain actions don't need to be displayed or history entries
	if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	crm_debug("No history injection for %s op on %s", operation, node);
	goto done; // Confirm action and update graph
	}

	if (action_rsc == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "Bad");
	free(node);
	return EPROTO;
	}

	/* A resource might be known by different names in the configuration and in
	* the action (for example, a clone instance). Grab the configuration name
	* (which is preferred when writing history), and if necessary, the instance
	* name.
	*/
	resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID);
	if (resource_config_name == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "No ID");
	free(node);
	return EPROTO;
	}
	resource = resource_config_name;
	if (pe_find_resource(fake_resource_list, resource) == NULL) {
	const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);

	if ((longname != NULL)
	&& (pe_find_resource(fake_resource_list, longname) != NULL)) {
	resource = longname;
	}
	}

	// Certain actions need to be displayed but don't need history entries
	if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE,
	PCMK_ACTION_META_DATA, NULL)) {
	out->message(out, "inject-rsc-action", resource, operation, node,
	(guint) 0);
	goto done; // Confirm action and update graph
	}

	rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
	rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);

	pcmk__scan_min_int(target_rc_s, &target_outcome, 0);

	CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
	cib_sync_call\|cib_scope_local) == pcmk_ok);

	// Ensure the action node is in the CIB
	uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	cib_node = pcmk__inject_node(fake_cib, node,
	((router_node == NULL)? uuid: node));
	free(uuid);
	CRM_ASSERT(cib_node != NULL);

	// Add a history entry for the action
	cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
	resource_config_name,
	rclass, rtype, rprovider);
	if (cib_resource == NULL) {
	crm_err("Could not simulate action %d history for resource %s",
	action->id, resource);
	free(node);
	free_xml(cib_node);
	return EINVAL;
	}

	// Simulate and display an executor event for the action result
	op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	target_outcome, "User-injected result");
	out->message(out, "inject-rsc-action", resource, op->op_type, node,
	op->interval_ms);

	// Check whether action is in a list of desired simulated failures
	for (const GList *iter = fake_op_fail_list;
	iter != NULL; iter = iter->next) {
	const char spec = (const char ) iter->data;
	char *key = NULL;
	const char *match_name = NULL;

	// Allow user to specify anonymous clone with or without instance number
	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource;
	}
	free(key);

	// If not found, try the resource's name in the configuration
	if ((match_name == NULL)
	&& (strcmp(resource, resource_config_name) != 0)) {

	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
	op->op_type, op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource_config_name;
	}
	free(key);
	}

	if (match_name == NULL) {
	continue; // This failed action entry doesn't match
	}

	// ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	if (rc != 1) {
	out->err(out, "Invalid failed operation '%s' "
	"(result code must be integer)", spec);
	continue; // Keep checking other list entries
	}

	out->info(out, "Pretending action %d failed with rc=%d",
	action->id, op->rc);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	graph->abort_priority = INFINITY;
	pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
	op->interval_ms, op->rc);
	break;
	}

	pcmk__inject_action_result(cib_resource, op, target_outcome);
	lrmd_free_event(op);
	rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(node);
	free_xml(cib_node);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a cluster action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Cluster action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-cluster-action", node, task, rsc);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a fencing action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Fencing action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_fencing_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *op = crm_meta_value(action->params, "stonith_action");
	char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);

	out->message(out, "inject-fencing-action", target, op);

	if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	int rc = pcmk_ok;
	GString *xpath = g_string_sized_new(512);

	// Set node state to offline
	xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
	false);

	CRM_ASSERT(cib_node != NULL);
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);
	rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	// Simulate controller clearing node's resource history and attributes
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']/" XML_CIB_TAG_LRM,
	NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	g_string_truncate(xpath, 0);
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']"
	"/" XML_TAG_TRANSIENT_NODEATTRS, NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	free_xml(cib_node);
	g_string_free(xpath, TRUE);
	}

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	free(target);
	return pcmk_rc_ok;
	}

	enum pcmk__graph_status
	pcmk__simulate_transition(pcmk_scheduler_t scheduler, cib_t cib,
	const GList *op_fail_list)
	{
	pcmk__graph_t *transition = NULL;
	enum pcmk__graph_status graph_rc;

	pcmk__graph_functions_t simulation_fns = {
	simulate_pseudo_action,
	simulate_resource_action,
	simulate_cluster_action,
	simulate_fencing_action,
	};

	out = scheduler->priv;

	fake_cib = cib;
	fake_op_fail_list = op_fail_list;

	if (!out->is_quiet(out)) {
	out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	}

	pcmk__set_graph_functions(&simulation_fns);
	transition = pcmk__unpack_graph(scheduler->graph, crm_system_name);
	pcmk__log_graph(LOG_DEBUG, transition);

	fake_resource_list = scheduler->resources;
	do {
	graph_rc = pcmk__execute_graph(transition);
	} while (graph_rc == pcmk__graph_active);
	fake_resource_list = NULL;

	if (graph_rc != pcmk__graph_complete) {
	out->err(out, "Transition failed: %s",
	pcmk__graph_status2text(graph_rc));
	pcmk__log_graph(LOG_ERR, transition);
	out->err(out, "An invalid transition was produced");
	}
	pcmk__free_graph(transition);

	if (!out->is_quiet(out)) {
	// If not quiet, we'll need the resulting CIB for later display
	xmlNode *cib_object = NULL;
	int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
	cib_sync_call\|cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	pe_reset_working_set(scheduler);
	scheduler->input = cib_object;
	out->end_list(out);
	}
	return graph_rc;
	}

	int
	pcmk__simulate(pcmk_scheduler_t scheduler, pcmk__output_t out,
	const pcmk_injections_t *injections, unsigned int flags,
	uint32_t section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	int printed = pcmk_rc_no_output;
	int rc = pcmk_rc_ok;
	xmlNodePtr input = NULL;
	cib_t *cib = NULL;

	rc = cib__signon_query(out, &cib, &input);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}

	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);

	if ((cib->variant == cib_native)
	&& pcmk_is_set(section_opts, pcmk_section_times)) {
	if (pcmk__our_nodename == NULL) {
	// Currently used only in the times section
	pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
	}
	scheduler->localhost = pcmk__our_nodename;
	}

	if (!out->is_quiet(out)) {
	const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	printed = out->message(out, "maint-mode", scheduler->flags);
	}

	if (scheduler->disabled_resources \|\| scheduler->blocked_resources) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	printed = out->info(out,
	"%d of %d resource instances DISABLED and "
	"%d BLOCKED from further action due to failure",
	scheduler->disabled_resources,
	scheduler->ninstances,
	scheduler->blocked_resources);
	}

	/* Most formatted output headers use caps for each word, but this one
	* only has the first word capitalized for compatibility with pcs.
	*/
	print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0),
	section_opts, "Current cluster status",
	(printed == pcmk_rc_ok));
	printed = pcmk_rc_ok;
	}

	// If the user requested any injections, handle them
	if ((injections->node_down != NULL)
	\|\| (injections->node_fail != NULL)
	\|\| (injections->node_up != NULL)
	\|\| (injections->op_inject != NULL)
	\|\| (injections->ticket_activate != NULL)
	\|\| (injections->ticket_grant != NULL)
	\|\| (injections->ticket_revoke != NULL)
	\|\| (injections->ticket_standby != NULL)
	\|\| (injections->watchdog != NULL)) {

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	pcmk__inject_scheduler_input(scheduler, cib, injections);
	printed = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
	if (rc != pcmk_rc_ok) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}

	cleanup_calculations(scheduler);
	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);
	}

	if (input_file != NULL) {
	rc = write_xml_file(input, input_file, FALSE);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}
	}

	if (pcmk_any_flags_set(flags, pcmk_sim_process \| pcmk_sim_simulate)) {
	pcmk__output_t *logger_out = NULL;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	if (pcmk_all_flags_set(scheduler->flags,
	pcmk_sched_output_scores
	\|pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL,
	"Assignment Scores and Utilization Information");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Assignment Scores");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Utilization Information");
	printed = pcmk_rc_ok;

	} else {
	rc = pcmk__log_output_new(&logger_out);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}
	pe__register_messages(logger_out);
	pcmk__register_lib_messages(logger_out);
	scheduler->priv = logger_out;
	}

	pcmk__schedule_actions(input, scheduler_flags, scheduler);

	if (logger_out == NULL) {
	out->end_list(out);
	} else {
	logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger_out);
	scheduler->priv = out;
	}

	input = NULL; /* Don't try and free it twice */

	if (graph_file != NULL) {
	rc = write_xml_file(scheduler->graph, graph_file, FALSE);
	if (rc < 0) {
	rc = pcmk_rc_graph_error;
	goto simulate_done;
	}
	}

	if (dot_file != NULL) {
	rc = write_sim_dotfile(scheduler, dot_file,
	pcmk_is_set(flags, pcmk_sim_all_actions),
	pcmk_is_set(flags, pcmk_sim_verbose));
	if (rc != pcmk_rc_ok) {
	rc = pcmk_rc_dot_error;
	goto simulate_done;
	}
	}

	if (!out->is_quiet(out)) {
	print_transition_summary(scheduler, printed == pcmk_rc_ok);
	}
	}

	rc = pcmk_rc_ok;

	if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
	goto simulate_done;
	}

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	if (pcmk__simulate_transition(scheduler, cib, injections->op_fail)
	!= pcmk__graph_complete) {
	rc = pcmk_rc_invalid_transition;
	}

	if (out->is_quiet(out)) {
	goto simulate_done;
	}

	set_effective_date(scheduler, true, use_date);

	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_output_scores);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_show_utilization);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}

	cluster_status(scheduler);
	print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status",
	true);

	simulate_done:
	cib__clean_up_connection(&cib);
	return rc;
	}

	int
	pcmk_simulate(xmlNodePtr xml, pcmk_scheduler_t scheduler,
	const pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__simulate(scheduler, out, injections, flags, section_opts,
	use_date, input_file, graph_file, dot_file);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	diff --git a/lib/pacemaker/pcmk_status.c b/lib/pacemaker/pcmk_status.c
	index 462d9c6a0e..7e97f0f66d 100644
	--- a/lib/pacemaker/pcmk_status.c
	+++ b/lib/pacemaker/pcmk_status.c
	@@ -1,381 +1,381 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stddef.h>
	#include <stdint.h>

	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/fencing/internal.h>
	#include <crm/pengine/internal.h>
	#include <crm/stonith-ng.h>
	#include <pacemaker.h>
	#include <pacemaker-internal.h>

	static stonith_t *
	fencing_connect(void)
	{
	stonith_t *st = stonith_api_new();
	int rc = pcmk_rc_ok;

	if (st == NULL) {
	return NULL;
	}

	rc = st->cmds->connect(st, crm_system_name, NULL);
	if (rc == pcmk_rc_ok) {
	return st;
	} else {
	stonith_api_delete(st);
	return NULL;
	}
	}

	/*!
	* \internal
	* \brief Output the cluster status given a fencer and CIB connection
	*
	* \param[in,out] out Output object
	* \param[in,out] stonith Fencer connection
	* \param[in,out] cib CIB connection
	* \param[in] current_cib Current CIB XML
	* \param[in] pcmkd_state \p pacemakerd state
	* \param[in] fence_history How much of the fencing history to output
	* \param[in] show Group of \p pcmk_section_e flags
	* \param[in] show_opts Group of \p pcmk_show_opt_e flags
	* \param[in] only_node If a node name or tag, include only the
	* matching node(s) (if any) in the output.
	* If \p "*" or \p NULL, include all nodes
	* in the output.
	* \param[in] only_rsc If a resource ID or tag, include only the
	* matching resource(s) (if any) in the
	* output. If \p "*" or \p NULL, include all
	* resources in the output.
	* \param[in] neg_location_prefix Prefix denoting a ban in a constraint ID
	* \param[in] simple_output Whether to use a simple output format.
	* Note: This is for use by \p crm_mon only
	* and is planned to be deprecated.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__output_cluster_status(pcmk__output_t out, stonith_t stonith, cib_t *cib,
	xmlNode *current_cib,
	enum pcmk_pacemakerd_state pcmkd_state,
	enum pcmk__fence_history fence_history,
	uint32_t show, uint32_t show_opts,
	const char only_node, const char only_rsc,
	const char *neg_location_prefix, bool simple_output)
	{
	xmlNode *cib_copy = copy_xml(current_cib);
	stonith_history_t *stonith_history = NULL;
	int history_rc = 0;
	pcmk_scheduler_t *scheduler = NULL;
	GList *unames = NULL;
	GList *resources = NULL;

	int rc = pcmk_rc_ok;

	if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
	cib__clean_up_connection(&cib);
	free_xml(cib_copy);
	rc = pcmk_rc_schema_validation;
	out->err(out, "Upgrade failed: %s", pcmk_rc_str(rc));
	return rc;
	}

	/* get the stonith-history if there is evidence we need it */
	if (fence_history != pcmk__fence_history_none) {
	history_rc = pcmk__get_fencing_history(stonith, &stonith_history,
	fence_history);
	}

	scheduler = pe_new_working_set();
	CRM_ASSERT(scheduler != NULL);
	- pe__set_working_set_flags(scheduler, pcmk_sched_no_compat);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_no_compat);

	scheduler->input = cib_copy;
	scheduler->priv = out;
	cluster_status(scheduler);

	if ((cib->variant == cib_native) && pcmk_is_set(show, pcmk_section_times)) {
	if (pcmk__our_nodename == NULL) {
	// Currently used only in the times section
	pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
	}
	scheduler->localhost = pcmk__our_nodename;
	}

	/* Unpack constraints if any section will need them
	* (tickets may be referenced in constraints but not granted yet,
	* and bans need negative location constraints) */
	if (pcmk_is_set(show, pcmk_section_bans)
	\|\| pcmk_is_set(show, pcmk_section_tickets)) {
	pcmk__unpack_constraints(scheduler);
	}

	unames = pe__build_node_name_list(scheduler, only_node);
	resources = pe__build_rsc_list(scheduler, only_rsc);

	/* Always print DC if NULL. */
	if (scheduler->dc_node == NULL) {
	show \|= pcmk_section_dc;
	}

	if (simple_output) {
	rc = pcmk__output_simple_status(out, scheduler);
	} else {
	out->message(out, "cluster-status",
	scheduler, pcmkd_state, pcmk_rc2exitc(history_rc),
	stonith_history, fence_history, show, show_opts,
	neg_location_prefix, unames, resources);
	}

	g_list_free_full(unames, free);
	g_list_free_full(resources, free);

	stonith_history_free(stonith_history);
	stonith_history = NULL;
	pe_free_working_set(scheduler);
	return rc;
	}

	int
	pcmk_status(xmlNodePtr *xml)
	{
	cib_t *cib = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	uint32_t show_opts = pcmk_show_pending
	\|pcmk_show_inactive_rscs
	\|pcmk_show_timing;

	cib = cib_new();

	if (cib == NULL) {
	return pcmk_rc_cib_corrupt;
	}

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	cib_delete(cib);
	return rc;
	}

	pcmk__register_lib_messages(out);
	pe__register_messages(out);
	stonith__register_messages(out);

	rc = pcmk__status(out, cib, pcmk__fence_history_full, pcmk_section_all,
	show_opts, NULL, NULL, NULL, false, 0);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);

	cib_delete(cib);
	return rc;
	}

	/*!
	* \internal
	* \brief Query and output the cluster status
	*
	* The operation is considered a success if we're able to get the \p pacemakerd
	* state. If possible, we'll also try to connect to the fencer and CIB and
	* output their respective status information.
	*
	* \param[in,out] out Output object
	* \param[in,out] cib CIB connection
	* \param[in] fence_history How much of the fencing history to output
	* \param[in] show Group of \p pcmk_section_e flags
	* \param[in] show_opts Group of \p pcmk_show_opt_e flags
	* \param[in] only_node If a node name or tag, include only the
	* matching node(s) (if any) in the output.
	* If \p "*" or \p NULL, include all nodes
	* in the output.
	* \param[in] only_rsc If a resource ID or tag, include only the
	* matching resource(s) (if any) in the
	* output. If \p "*" or \p NULL, include all
	* resources in the output.
	* \param[in] neg_location_prefix Prefix denoting a ban in a constraint ID
	* \param[in] simple_output Whether to use a simple output format.
	* Note: This is for use by \p crm_mon only
	* and is planned to be deprecated.
	* \param[in] timeout_ms How long to wait for a reply from the
	* \p pacemakerd API. If 0,
	* \p pcmk_ipc_dispatch_sync will be used.
	* If positive, \p pcmk_ipc_dispatch_main
	* will be used, and a new mainloop will be
	* created for this purpose (freed before
	* return).
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__status(pcmk__output_t out, cib_t cib,
	enum pcmk__fence_history fence_history, uint32_t show,
	uint32_t show_opts, const char only_node, const char only_rsc,
	const char *neg_location_prefix, bool simple_output,
	unsigned int timeout_ms)
	{
	xmlNode *current_cib = NULL;
	int rc = pcmk_rc_ok;
	stonith_t *stonith = NULL;
	enum pcmk_pacemakerd_state pcmkd_state = pcmk_pacemakerd_state_invalid;
	time_t last_updated = 0;

	if (cib == NULL) {
	return ENOTCONN;
	}

	if (cib->variant == cib_native) {
	rc = pcmk__pacemakerd_status(out, crm_system_name, timeout_ms, false,
	&pcmkd_state);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	last_updated = time(NULL);

	switch (pcmkd_state) {
	case pcmk_pacemakerd_state_running:
	case pcmk_pacemakerd_state_shutting_down:
	case pcmk_pacemakerd_state_remote:
	/* Fencer and CIB may still be available while shutting down or
	* running on a Pacemaker Remote node
	*/
	break;
	default:
	// Fencer and CIB are definitely unavailable
	out->message(out, "pacemakerd-health",
	NULL, pcmkd_state, NULL, last_updated);
	return rc;
	}

	if (fence_history != pcmk__fence_history_none) {
	stonith = fencing_connect();
	}
	}

	rc = cib__signon_query(out, &cib, &current_cib);
	if (rc != pcmk_rc_ok) {
	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	// Invalid at this point means we didn't query the pcmkd state
	out->message(out, "pacemakerd-health",
	NULL, pcmkd_state, NULL, last_updated);
	}
	goto done;
	}

	rc = pcmk__output_cluster_status(out, stonith, cib, current_cib,
	pcmkd_state, fence_history, show,
	show_opts, only_node, only_rsc,
	neg_location_prefix, simple_output);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Error outputting status info from the fencer or CIB");
	}

	done:
	stonith_api_delete(stonith);
	free_xml(current_cib);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Output cluster status in Nagios Plugin format
	*
	* \param[in,out] out Output object
	* \param[in] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note This is for a deprecated crm_mon option and should be called only for
	* that.
	*/
	int
	pcmk__output_simple_status(pcmk__output_t *out,
	const pcmk_scheduler_t *scheduler)
	{
	int nodes_online = 0;
	int nodes_standby = 0;
	int nodes_maint = 0;
	GString *offline_nodes = NULL;
	bool no_dc = false;
	bool offline = false;
	bool has_warnings = false;

	if (scheduler->dc_node == NULL) {
	has_warnings = true;
	no_dc = true;
	}

	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (node->details->standby && node->details->online) {
	nodes_standby++;
	} else if (node->details->maintenance && node->details->online) {
	nodes_maint++;
	} else if (node->details->online) {
	nodes_online++;
	} else {
	pcmk__add_word(&offline_nodes, 1024, "offline node:");
	pcmk__add_word(&offline_nodes, 0, pe__node_name(node));
	has_warnings = true;
	offline = true;
	}
	}

	if (has_warnings) {
	out->info(out, "CLUSTER WARN: %s%s%s",
	no_dc ? "No DC" : "",
	no_dc && offline ? ", " : "",
	(offline? (const char *) offline_nodes->str : ""));

	if (offline_nodes != NULL) {
	g_string_free(offline_nodes, TRUE);
	}

	} else {
	char *nodes_standby_s = NULL;
	char *nodes_maint_s = NULL;

	if (nodes_standby > 0) {
	nodes_standby_s = crm_strdup_printf(", %d standby node%s",
	nodes_standby,
	pcmk__plural_s(nodes_standby));
	}

	if (nodes_maint > 0) {
	nodes_maint_s = crm_strdup_printf(", %d maintenance node%s",
	nodes_maint,
	pcmk__plural_s(nodes_maint));
	}

	out->info(out, "CLUSTER OK: %d node%s online%s%s, "
	"%d resource instance%s configured",
	nodes_online, pcmk__plural_s(nodes_online),
	nodes_standby_s != NULL ? nodes_standby_s : "",
	nodes_maint_s != NULL ? nodes_maint_s : "",
	scheduler->ninstances, pcmk__plural_s(scheduler->ninstances));

	free(nodes_standby_s);
	free(nodes_maint_s);
	}

	if (has_warnings) {
	return pcmk_rc_error;
	} else {
	return pcmk_rc_ok;
	}
	/* coverity[leaked_storage] False positive */
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 75bcd56527..50cfaa6de9 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1202 +1,1202 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pcmk_node_t active_node(const pcmk_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	pcmk_rsc_methods_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pcmk_rsc_variant_primitive;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pcmk_rsc_variant_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pcmk_rsc_variant_bundle;
	}

	return pcmk_rsc_variant_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
	pe_rule_eval_data_t *rule_data,
	GHashTable meta_hash, pcmk_scheduler_t scheduler)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pcmk_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS, rule_data,
	parent_orig_meta, NULL, FALSE, scheduler);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	.standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	.provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	.agent = crm_element_value(rsc->xml, XML_ATTR_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, scheduler);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, scheduler);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE,
	scheduler);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, "name");
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//" XML_CIB_TAG_RESOURCES,
	scheduler->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pcmk__config_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	PCMK_XA_ID, template_ref);
	if (template == NULL) {
	pcmk__config_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, PCMK_XA_ID, id);
	crm_xml_add(new_xml, PCMK__META_CLONE_INSTANCE_NUM,
	crm_element_value(xml_obj, PCMK__META_CLONE_INSTANCE_NUM));

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	#if 0 /* Disable multi-level templates for now */
	if (!unpack_template(new_xml, expanded_xml, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}
	#endif

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for processing resource list "
	"of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	if (add_tag_ref(scheduler->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pcmk_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	PCMK_META_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
	/* @TODO in some future version, pcmk__warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(PCMK_META_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p scheduler is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (scheduler == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, scheduler);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pcmk_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pe__set_resource_flags(rsc, pcmk_rsc_needs_fencing
	\|pcmk_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pcmk_resource_t *rsc)
	{
	const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pcmk__warn_once(pcmk__wo_upstart,
	"Support for Upstart resources (such as %s) is "
	"deprecated and will be removed in a future release",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pcmk__warn_once(pcmk__wo_nagios,
	"Support for Nagios resources (such as %s) is "
	"deprecated and will be removed in a future release",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pcmk_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pcmk_resource_t *rsc,
	pcmk_resource_t parent, pcmk_scheduler_t scheduler)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = scheduler->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pcmk_resource_t));
	if (*rsc == NULL) {
	pcmk__sched_err("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = scheduler;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, scheduler->input);

	(rsc)->variant = get_resource_type((const char ) (*rsc)->xml->name);
	if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
	pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
	id, (*rsc)->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, PCMK__META_CLONE_INSTANCE_NUM);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, PCMK__META_CLONE_INSTANCE_NUM, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, scheduler);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, scheduler); // \deprecated

	(*rsc)->flags = 0;
	pe__set_resource_flags(*rsc, pcmk_rsc_runnable\|pcmk_rsc_unassigned);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = pcmk_role_stopped;
	(*rsc)->next_role = pcmk_role_unknown;

	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_NOTIFY);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, PCMK__META_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* PCMK_META_ALLOW_MIGRATE=false.
	*/
	pe__set_resource_flags(*rsc, pcmk_rsc_migratable);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_IS_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_managed);
	} else {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MAINTENANCE);
	if (crm_is_true(value)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pe__clear_resource_flags(*rsc, pcmk_rsc_managed);
	pe__set_resource_flags(*rsc, pcmk_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_GLOBALLY_UNIQUE);
	if (crm_is_true(value)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pe__set_resource_flags(*rsc, pcmk_rsc_promotable);
	}
	} else {
	pe__set_resource_flags(*rsc, pcmk_rsc_unique);
	}

	// @COMPAT Deprecated meta-attribute
	value = g_hash_table_lookup((*rsc)->meta, PCMK__META_RESTART_TYPE);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
	(*rsc)->id);
	pcmk__warn_once(pcmk__wo_restart_type,
	"Support for " PCMK__META_RESTART_TYPE " is deprecated "
	"and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MULTIPLE_ACTIVE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_stop;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_block;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_warn("%s is not a valid value for "
	PCMK_META_MULTIPLE_ACTIVE
	", using default of \"stop_start\"", value);
	}
	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: stop/start",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_RESOURCE_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pcmk__warn_once(pcmk__wo_neg_threshold,
	PCMK_META_MIGRATION_THRESHOLD
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	pe__set_resource_flags(*rsc, pcmk_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	guint interval_ms = 0U;

	// Stored as seconds
	pcmk_parse_interval_spec(value, &interval_ms);
	(*rsc)->failure_timeout = (int) (interval_ms / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, scheduler);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	pcmk_parse_interval_spec(value, &((*rsc)->remote_reconnect_ms));

	/* We want to override any default failure_timeout in use when remote
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use.
	*/
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pcmk__rsc_trace(rsc, "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != pcmk_role_unknown? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default",
	scheduler);
	}

	pcmk__rsc_trace(rsc, "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, scheduler);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pcmk_resource_t child, pcmk_resource_t rsc)
	{
	pcmk_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pcmk_resource_t *
	uber_parent(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->parent != NULL)
	&& (parent->parent->variant != pcmk_rsc_variant_bundle)) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pcmk_resource_t *
	pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
	{
	const pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle
	&& (parent->parent->variant == pcmk_rsc_variant_bundle)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pcmk_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pcmk__rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {

	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pcmk__rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of pcmk_rsc_methods_t:active_node()
	static pcmk_node_t *
	active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pcmk_node_t *
	pe__find_active_requires(const pcmk_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pcmk_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pcmk_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role > pcmk_role_stopped)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pcmk_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role),
	why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/status.c b/lib/pengine/status.c
	index fa2fa3f9ef..ec88d1cbe6 100644
	--- a/lib/pengine/status.c
	+++ b/lib/pengine/status.c
	@@ -1,484 +1,484 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	/*!
	* \brief Create a new object to hold scheduler data
	*
	* \return New, initialized scheduler data on success, else NULL (and set errno)
	* \note Only pcmk_scheduler_t objects created with this function (as opposed
	* to statically declared or directly allocated) should be used with the
	* functions in this library, to allow for future extensions to the
	* data type. The caller is responsible for freeing the memory with
	* pe_free_working_set() when the instance is no longer needed.
	*/
	pcmk_scheduler_t *
	pe_new_working_set(void)
	{
	pcmk_scheduler_t *scheduler = calloc(1, sizeof(pcmk_scheduler_t));

	if (scheduler != NULL) {
	set_working_set_defaults(scheduler);
	}
	return scheduler;
	}

	/*!
	* \brief Free scheduler data
	*
	* \param[in,out] scheduler Scheduler data to free
	*/
	void
	pe_free_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler != NULL) {
	pe_reset_working_set(scheduler);
	scheduler->priv = NULL;
	free(scheduler);
	}
	}

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependencies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependencies)
	*/
	gboolean
	cluster_status(pcmk_scheduler_t * scheduler)
	{
	xmlNode *section = NULL;

	if ((scheduler == NULL) \|\| (scheduler->input == NULL)) {
	return FALSE;
	}

	crm_trace("Beginning unpack");

	if (scheduler->failed != NULL) {
	free_xml(scheduler->failed);
	}
	scheduler->failed = create_xml_node(NULL, "failed-ops");

	if (scheduler->now == NULL) {
	scheduler->now = crm_time_new(NULL);
	}

	if (scheduler->dc_uuid == NULL) {
	scheduler->dc_uuid = crm_element_value_copy(scheduler->input,
	XML_ATTR_DC_UUID);
	}

	if (pcmk__xe_attr_is_true(scheduler->input, PCMK_XA_HAVE_QUORUM)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_quorate);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_quorate);
	} else {
	- pe__clear_working_set_flags(scheduler, pcmk_sched_quorate);
	+ pcmk__clear_scheduler_flags(scheduler, pcmk_sched_quorate);
	}

	scheduler->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
	scheduler->input, LOG_NEVER);
	scheduler->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
	scheduler->input, LOG_NEVER);

	section = get_xpath_object("//" XML_CIB_TAG_CRMCONFIG, scheduler->input,
	LOG_TRACE);
	unpack_config(section, scheduler);

	if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_location_only\|pcmk_sched_quorate)
	&& (scheduler->no_quorum_policy != pcmk_no_quorum_ignore)) {
	pcmk__sched_warn("Fencing and resource management disabled "
	"due to lack of quorum");
	}

	section = get_xpath_object("//" XML_CIB_TAG_NODES, scheduler->input,
	LOG_TRACE);
	unpack_nodes(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_RESOURCES, scheduler->input,
	LOG_TRACE);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	unpack_remote_nodes(section, scheduler);
	}
	unpack_resources(section, scheduler);

	section = get_xpath_object("//" XML_CIB_TAG_TAGS, scheduler->input,
	LOG_NEVER);
	unpack_tags(section, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	section = get_xpath_object("//"XML_CIB_TAG_STATUS, scheduler->input,
	LOG_TRACE);
	unpack_status(section, scheduler);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_no_counts)) {
	for (GList *item = scheduler->resources; item != NULL;
	item = item->next) {
	((pcmk_resource_t *) (item->data))->fns->count(item->data);
	}
	crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	- pe__set_working_set_flags(scheduler, pcmk_sched_have_status);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_status);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Free a list of pcmk_resource_t
	*
	* \param[in,out] resources List to free
	*
	* \note When the scheduler's resource list is freed, that includes the original
	* storage for the uname and id of any Pacemaker Remote nodes in the
	* scheduler's node list, so take care not to use those afterward.
	* \todo Refactor pcmk_node_t to strdup() the node name.
	*/
	static void
	pe_free_resources(GList *resources)
	{
	pcmk_resource_t *rsc = NULL;
	GList *iterator = resources;

	while (iterator != NULL) {
	rsc = (pcmk_resource_t *) iterator->data;
	iterator = iterator->next;
	rsc->fns->free(rsc);
	}
	if (resources != NULL) {
	g_list_free(resources);
	}
	}

	static void
	pe_free_actions(GList *actions)
	{
	GList *iterator = actions;

	while (iterator != NULL) {
	pe_free_action(iterator->data);
	iterator = iterator->next;
	}
	if (actions != NULL) {
	g_list_free(actions);
	}
	}

	static void
	pe_free_nodes(GList *nodes)
	{
	for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
	pcmk_node_t node = (pcmk_node_t ) iterator->data;

	// Shouldn't be possible, but to be safe ...
	if (node == NULL) {
	continue;
	}
	if (node->details == NULL) {
	free(node);
	continue;
	}

	/* This is called after pe_free_resources(), which means that we can't
	* use node->details->uname for Pacemaker Remote nodes.
	*/
	crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
	"(guest or remote)" : pe__node_name(node)));

	if (node->details->attrs != NULL) {
	g_hash_table_destroy(node->details->attrs);
	}
	if (node->details->utilization != NULL) {
	g_hash_table_destroy(node->details->utilization);
	}
	if (node->details->digest_cache != NULL) {
	g_hash_table_destroy(node->details->digest_cache);
	}
	g_list_free(node->details->running_rsc);
	g_list_free(node->details->allocated_rsc);
	free(node->details);
	free(node);
	}
	if (nodes != NULL) {
	g_list_free(nodes);
	}
	}

	static void
	pe__free_ordering(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__action_relation_t *order = iterator->data;

	iterator = iterator->next;

	free(order->task1);
	free(order->task2);
	free(order);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	static void
	pe__free_location(GList *constraints)
	{
	GList *iterator = constraints;

	while (iterator != NULL) {
	pcmk__location_t *cons = iterator->data;

	iterator = iterator->next;

	g_list_free_full(cons->nodes, free);
	free(cons->id);
	free(cons);
	}
	if (constraints != NULL) {
	g_list_free(constraints);
	}
	}

	/*!
	* \brief Reset scheduler data to defaults without freeing it or constraints
	*
	* \param[in,out] scheduler Scheduler data to reset
	*
	* \deprecated This function is deprecated as part of the API;
	* pe_reset_working_set() should be used instead.
	*/
	void
	cleanup_calculations(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	- pe__clear_working_set_flags(scheduler, pcmk_sched_have_status);
	+ pcmk__clear_scheduler_flags(scheduler, pcmk_sched_have_status);
	if (scheduler->config_hash != NULL) {
	g_hash_table_destroy(scheduler->config_hash);
	}

	if (scheduler->singletons != NULL) {
	g_hash_table_destroy(scheduler->singletons);
	}

	if (scheduler->tickets) {
	g_hash_table_destroy(scheduler->tickets);
	}

	if (scheduler->template_rsc_sets) {
	g_hash_table_destroy(scheduler->template_rsc_sets);
	}

	if (scheduler->tags) {
	g_hash_table_destroy(scheduler->tags);
	}

	free(scheduler->dc_uuid);

	crm_trace("deleting resources");
	pe_free_resources(scheduler->resources);

	crm_trace("deleting actions");
	pe_free_actions(scheduler->actions);

	crm_trace("deleting nodes");
	pe_free_nodes(scheduler->nodes);

	pe__free_param_checks(scheduler);
	g_list_free(scheduler->stop_needed);
	free_xml(scheduler->graph);
	crm_time_free(scheduler->now);
	free_xml(scheduler->input);
	free_xml(scheduler->failed);

	set_working_set_defaults(scheduler);

	CRM_CHECK(scheduler->ordering_constraints == NULL,;
	);
	CRM_CHECK(scheduler->placement_constraints == NULL,;
	);
	}

	/*!
	* \brief Reset scheduler data to default state without freeing it
	*
	* \param[in,out] scheduler Scheduler data to reset
	*/
	void
	pe_reset_working_set(pcmk_scheduler_t *scheduler)
	{
	if (scheduler == NULL) {
	return;
	}

	crm_trace("Deleting %d ordering constraints",
	g_list_length(scheduler->ordering_constraints));
	pe__free_ordering(scheduler->ordering_constraints);
	scheduler->ordering_constraints = NULL;

	crm_trace("Deleting %d location constraints",
	g_list_length(scheduler->placement_constraints));
	pe__free_location(scheduler->placement_constraints);
	scheduler->placement_constraints = NULL;

	crm_trace("Deleting %d colocation constraints",
	g_list_length(scheduler->colocation_constraints));
	g_list_free_full(scheduler->colocation_constraints, free);
	scheduler->colocation_constraints = NULL;

	crm_trace("Deleting %d ticket constraints",
	g_list_length(scheduler->ticket_constraints));
	g_list_free_full(scheduler->ticket_constraints, free);
	scheduler->ticket_constraints = NULL;

	cleanup_calculations(scheduler);
	}

	void
	set_working_set_defaults(pcmk_scheduler_t *scheduler)
	{
	void *priv = scheduler->priv;

	memset(scheduler, 0, sizeof(pcmk_scheduler_t));

	scheduler->priv = priv;
	scheduler->order_id = 1;
	scheduler->action_id = 1;
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;

	scheduler->flags = 0x0ULL;

	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_symmetric_cluster
	\|pcmk_sched_stop_removed_resources
	\|pcmk_sched_cancel_removed_actions);
	if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, "true")) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_concurrent_fencing);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_concurrent_fencing);
	}
	}

	pcmk_resource_t *
	pe_find_resource(GList rsc_list, const char id)
	{
	return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
	}

	pcmk_resource_t *
	pe_find_resource_with_flags(GList rsc_list, const char id, enum pe_find flags)
	{
	GList *rIter = NULL;

	for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
	pcmk_resource_t *parent = rIter->data;

	pcmk_resource_t *match =
	parent->fns->find_rsc(parent, id, NULL, flags);
	if (match != NULL) {
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \brief Find a node by name or ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id If not NULL, ID of node to find
	* \param[in] node_name If not NULL, name of node to find
	*
	* \return Node from \p nodes that matches \p id if any,
	* otherwise node from \p nodes that matches \p uname if any,
	* otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_any(const GList nodes, const char id, const char *uname)
	{
	pcmk_node_t *match = NULL;

	if (id != NULL) {
	match = pe_find_node_id(nodes, id);
	}
	if ((match == NULL) && (uname != NULL)) {
	match = pe_find_node(nodes, uname);
	}
	return match;
	}

	/*!
	* \brief Find a node by ID in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] id ID of node to find
	*
	* \return Node from \p nodes that matches \p id if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node_id(const GList nodes, const char id)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* @TODO Whether node IDs should be considered case-sensitive should
	* probably depend on the node type, so functionizing the comparison
	* would be worthwhile
	*/
	if (pcmk__str_eq(node->details->id, id, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}

	/*!
	* \brief Find a node by name in a list of nodes
	*
	* \param[in] nodes List of nodes (as pcmk_node_t*)
	* \param[in] node_name Name of node to find
	*
	* \return Node from \p nodes that matches \p node_name if any, otherwise NULL
	*/
	pcmk_node_t *
	pe_find_node(const GList nodes, const char node_name)
	{
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	if (pcmk__str_eq(node->details->uname, node_name, pcmk__str_casei)) {
	return node;
	}
	}
	return NULL;
	}
	diff --git a/lib/pengine/tests/native/native_find_rsc_test.c b/lib/pengine/tests/native/native_find_rsc_test.c
	index 996666777d..1de10bf971 100644
	--- a/lib/pengine/tests/native/native_find_rsc_test.c
	+++ b/lib/pengine/tests/native/native_find_rsc_test.c
	@@ -1,917 +1,917 @@
	/*
	* Copyright 2022-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/scheduler.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/status.h>

	xmlNode *input = NULL;
	pcmk_scheduler_t *scheduler = NULL;

	pcmk_node_t cluster01, cluster02, *httpd_bundle_0;
	pcmk_resource_t exim_group, inactive_group;
	pcmk_resource_t promotable_clone, inactive_clone;
	pcmk_resource_t httpd_bundle, mysql_clone_group;

	static int
	setup(void **state) {
	char *path = NULL;

	crm_xml_init();

	path = crm_strdup_printf("%s/crm_mon.xml", getenv("PCMK_CTS_CLI_DIR"));
	input = filename2xml(path);
	free(path);

	if (input == NULL) {
	return 1;
	}

	scheduler = pe_new_working_set();

	if (scheduler == NULL) {
	return 1;
	}

	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	scheduler->input = input;

	cluster_status(scheduler);

	/* Get references to the cluster nodes so we don't have to find them repeatedly. */
	cluster01 = pe_find_node(scheduler->nodes, "cluster01");
	cluster02 = pe_find_node(scheduler->nodes, "cluster02");
	httpd_bundle_0 = pe_find_node(scheduler->nodes, "httpd-bundle-0");

	/* Get references to several resources we use frequently. */
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "exim-group") == 0) {
	exim_group = rsc;
	} else if (strcmp(rsc->id, "httpd-bundle") == 0) {
	httpd_bundle = rsc;
	} else if (strcmp(rsc->id, "inactive-clone") == 0) {
	inactive_clone = rsc;
	} else if (strcmp(rsc->id, "inactive-group") == 0) {
	inactive_group = rsc;
	} else if (strcmp(rsc->id, "mysql-clone-group") == 0) {
	mysql_clone_group = rsc;
	} else if (strcmp(rsc->id, "promotable-clone") == 0) {
	promotable_clone = rsc;
	}
	}

	return 0;
	}

	static int
	teardown(void **state) {
	pe_free_working_set(scheduler);

	return 0;
	}

	static void
	bad_args(void **state) {
	pcmk_resource_t *rsc = g_list_first(scheduler->resources)->data;
	char *id = rsc->id;
	char *name = NULL;

	assert_non_null(rsc);

	assert_null(native_find_rsc(NULL, "dummy", NULL, 0));
	assert_null(native_find_rsc(rsc, NULL, NULL, 0));

	/* No resources exist with these names. */
	name = crm_strdup_printf("%sX", rsc->id);
	assert_null(native_find_rsc(rsc, name, NULL, 0));
	free(name);

	name = crm_strdup_printf("x%s", rsc->id);
	assert_null(native_find_rsc(rsc, name, NULL, 0));
	free(name);

	name = g_ascii_strup(rsc->id, -1);
	assert_null(native_find_rsc(rsc, name, NULL, 0));
	g_free(name);

	/* Fails because resource ID is NULL. */
	rsc->id = NULL;
	assert_null(native_find_rsc(rsc, id, NULL, 0));
	rsc->id = id;
	}

	static void
	primitive_rsc(void **state) {
	pcmk_resource_t *dummy = NULL;

	/* Find the "dummy" resource, which is the only one with that ID in the set. */
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "dummy") == 0) {
	dummy = rsc;
	break;
	}
	}

	assert_non_null(dummy);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(dummy, native_find_rsc(dummy, "dummy", NULL, 0));
	assert_ptr_equal(dummy,
	native_find_rsc(dummy, "dummy", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(dummy, "dummy", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(dummy, "dummy", cluster02,
	pcmk_rsc_match_clone_only));

	/* Fails because dummy is not running on cluster01, even with the right flags. */
	assert_null(native_find_rsc(dummy, "dummy", cluster01,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(dummy, "dummy", cluster02, 0));

	/* Passes because dummy is running on cluster02. */
	assert_ptr_equal(dummy,
	native_find_rsc(dummy, "dummy", cluster02,
	pcmk_rsc_match_current_node));
	}

	static void
	group_rsc(void **state) {
	assert_non_null(exim_group);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(exim_group, native_find_rsc(exim_group, "exim-group", NULL, 0));
	assert_ptr_equal(exim_group,
	native_find_rsc(exim_group, "exim-group", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(exim_group, "exim-group", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(exim_group, "exim-group", cluster01,
	pcmk_rsc_match_clone_only));

	/* Fails because none of exim-group's children are running on cluster01, even with the right flags. */
	assert_null(native_find_rsc(exim_group, "exim-group", cluster01,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(exim_group, "exim-group", cluster01, 0));

	/* Passes because one of exim-group's children is running on cluster02. */
	assert_ptr_equal(exim_group,
	native_find_rsc(exim_group, "exim-group", cluster02,
	pcmk_rsc_match_current_node));
	}

	static void
	inactive_group_rsc(void **state) {
	assert_non_null(inactive_group);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(inactive_group, native_find_rsc(inactive_group, "inactive-group", NULL, 0));
	assert_ptr_equal(inactive_group,
	native_find_rsc(inactive_group, "inactive-group", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(inactive_group, "inactive-group", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(inactive_group, "inactive-group", cluster01,
	pcmk_rsc_match_clone_only));

	/* Fails because none of inactive-group's children are running. */
	assert_null(native_find_rsc(inactive_group, "inactive-group", cluster01,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(inactive_group, "inactive-group", cluster02,
	pcmk_rsc_match_current_node));
	}

	static void
	group_member_rsc(void **state) {
	pcmk_resource_t *public_ip = NULL;

	/* Find the "Public-IP" resource, a member of "exim-group". */
	for (GList *iter = exim_group->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "Public-IP") == 0) {
	public_ip = rsc;
	break;
	}
	}

	assert_non_null(public_ip);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(public_ip, native_find_rsc(public_ip, "Public-IP", NULL, 0));
	assert_ptr_equal(public_ip,
	native_find_rsc(public_ip, "Public-IP", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(public_ip, "Public-IP", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(public_ip, "Public-IP", cluster02,
	pcmk_rsc_match_clone_only));

	/* Fails because Public-IP is not running on cluster01, even with the right flags. */
	assert_null(native_find_rsc(public_ip, "Public-IP", cluster01,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(public_ip, "Public-IP", cluster02, 0));

	/* Passes because Public-IP is running on cluster02. */
	assert_ptr_equal(public_ip,
	native_find_rsc(public_ip, "Public-IP", cluster02,
	pcmk_rsc_match_current_node));
	}

	static void
	inactive_group_member_rsc(void **state) {
	pcmk_resource_t *inactive_dummy_1 = NULL;

	/* Find the "inactive-dummy-1" resource, a member of "inactive-group". */
	for (GList *iter = inactive_group->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "inactive-dummy-1") == 0) {
	inactive_dummy_1 = rsc;
	break;
	}
	}

	assert_non_null(inactive_dummy_1);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(inactive_dummy_1, native_find_rsc(inactive_dummy_1, "inactive-dummy-1", NULL, 0));
	assert_ptr_equal(inactive_dummy_1,
	native_find_rsc(inactive_dummy_1, "inactive-dummy-1", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(inactive_dummy_1, "inactive-dummy-1", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(inactive_dummy_1, "inactive-dummy-1", cluster01,
	pcmk_rsc_match_clone_only));

	/* Fails because inactive-dummy-1 is not running. */
	assert_null(native_find_rsc(inactive_dummy_1, "inactive-dummy-1", cluster01,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(inactive_dummy_1, "inactive-dummy-1", cluster02,
	pcmk_rsc_match_current_node));
	}

	static void
	clone_rsc(void **state) {
	assert_non_null(promotable_clone);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(promotable_clone, native_find_rsc(promotable_clone, "promotable-clone", NULL, 0));
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone", NULL,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone", NULL,
	pcmk_rsc_match_clone_only));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(promotable_clone, "promotable-clone", cluster01, 0));

	/* Passes because one of ping-clone's children is running on cluster01. */
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone",
	cluster01, pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(promotable_clone, "promotable-clone", cluster02, 0));

	/* Passes because one of ping_clone's children is running on cluster02. */
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone",
	cluster02, pcmk_rsc_match_current_node));

	// Passes for previous reasons, plus includes pcmk_rsc_match_clone_only
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone",
	cluster01,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_clone,
	native_find_rsc(promotable_clone, "promotable-clone",
	cluster02,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node));
	}

	static void
	inactive_clone_rsc(void **state) {
	assert_non_null(inactive_clone);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(inactive_clone, native_find_rsc(inactive_clone, "inactive-clone", NULL, 0));
	assert_ptr_equal(inactive_clone,
	native_find_rsc(inactive_clone, "inactive-clone", NULL,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(inactive_clone,
	native_find_rsc(inactive_clone, "inactive-clone", NULL,
	pcmk_rsc_match_clone_only));

	/* Fails because none of inactive-clone's children are running. */
	assert_null(native_find_rsc(inactive_clone, "inactive-clone", cluster01,
	pcmk_rsc_match_current_node
	\|pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(inactive_clone, "inactive-clone", cluster02,
	pcmk_rsc_match_current_node
	\|pcmk_rsc_match_clone_only));
	}

	static void
	clone_instance_rsc(void **state) {
	pcmk_resource_t *promotable_0 = NULL;
	pcmk_resource_t *promotable_1 = NULL;

	/* Find the "promotable-rsc:0" and "promotable-rsc:1" resources, members of "promotable-clone". */
	for (GList *iter = promotable_clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "promotable-rsc:0") == 0) {
	promotable_0 = rsc;
	} else if (strcmp(rsc->id, "promotable-rsc:1") == 0) {
	promotable_1 = rsc;
	}
	}

	assert_non_null(promotable_0);
	assert_non_null(promotable_1);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(promotable_0, native_find_rsc(promotable_0, "promotable-rsc:0", NULL, 0));
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc:0", NULL,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1, native_find_rsc(promotable_1, "promotable-rsc:1", NULL, 0));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc:1", NULL,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(promotable_0, "promotable-rsc:0", cluster02, 0));
	assert_null(native_find_rsc(promotable_1, "promotable-rsc:1", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc:0",
	cluster02, pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_0, "promotable-rsc:0", cluster01,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc:1",
	cluster01, pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_1, "promotable-rsc:1", cluster02,
	pcmk_rsc_match_current_node));

	/* Passes because NULL was passed for node and primitive name was given, with correct flags. */
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", NULL,
	pcmk_rsc_match_clone_only));

	// Passes because pcmk_rsc_match_basename matches any instance's base name
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", NULL,
	pcmk_rsc_match_basename));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc", NULL,
	pcmk_rsc_match_basename));

	// Passes because pcmk_rsc_match_anon_basename matches
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", NULL,
	pcmk_rsc_match_anon_basename));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc", NULL,
	pcmk_rsc_match_anon_basename));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_0, "promotable-rsc", cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_0, "promotable-rsc", cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc", cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc", cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_1, "promotable-rsc", cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_1, "promotable-rsc", cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));

	/* Fails because incorrect flags were given along with primitive name. */
	assert_null(native_find_rsc(promotable_0, "promotable-rsc", NULL,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(promotable_1, "promotable-rsc", NULL,
	pcmk_rsc_match_current_node));

	/* And then we check failure possibilities again, except passing promotable_clone
	* instead of promotable_X as the first argument to native_find_rsc.
	*/

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(promotable_clone, "promotable-rsc:0", cluster02, 0));
	assert_null(native_find_rsc(promotable_clone, "promotable-rsc:1", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_clone, "promotable-rsc:0",
	cluster02, pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_clone, "promotable-rsc",
	cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_clone, "promotable-rsc",
	cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_clone, "promotable-rsc:1",
	cluster01, pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_clone, "promotable-rsc",
	cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_clone, "promotable-rsc",
	cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	}

	static void
	renamed_rsc(void **state) {
	pcmk_resource_t *promotable_0 = NULL;
	pcmk_resource_t *promotable_1 = NULL;

	/* Find the "promotable-rsc:0" and "promotable-rsc:1" resources, members of "promotable-clone". */
	for (GList *iter = promotable_clone->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "promotable-rsc:0") == 0) {
	promotable_0 = rsc;
	} else if (strcmp(rsc->id, "promotable-rsc:1") == 0) {
	promotable_1 = rsc;
	}
	}

	assert_non_null(promotable_0);
	assert_non_null(promotable_1);

	// Passes because pcmk_rsc_match_history means base name matches clone_name
	assert_ptr_equal(promotable_0,
	native_find_rsc(promotable_0, "promotable-rsc", NULL,
	pcmk_rsc_match_history));
	assert_ptr_equal(promotable_1,
	native_find_rsc(promotable_1, "promotable-rsc", NULL,
	pcmk_rsc_match_history));
	}

	static void
	bundle_rsc(void **state) {
	assert_non_null(httpd_bundle);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(httpd_bundle, native_find_rsc(httpd_bundle, "httpd-bundle", NULL, 0));
	assert_ptr_equal(httpd_bundle,
	native_find_rsc(httpd_bundle, "httpd-bundle", NULL,
	pcmk_rsc_match_current_node));

	/* Fails because resource is not a clone (nor cloned). */
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle", NULL,
	pcmk_rsc_match_clone_only));
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle", cluster01,
	pcmk_rsc_match_clone_only));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle", cluster01, 0));

	/* Passes because one of httpd_bundle's children is running on cluster01. */
	assert_ptr_equal(httpd_bundle,
	native_find_rsc(httpd_bundle, "httpd-bundle", cluster01,
	pcmk_rsc_match_current_node));
	}

	static bool
	bundle_first_replica(pcmk__bundle_replica_t replica, void user_data)
	{
	pcmk_resource_t *ip_0 = replica->ip;
	pcmk_resource_t *child_0 = replica->child;
	pcmk_resource_t *container_0 = replica->container;
	pcmk_resource_t *remote_0 = replica->remote;

	assert_non_null(ip_0);
	assert_non_null(child_0);
	assert_non_null(container_0);
	assert_non_null(remote_0);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(ip_0, native_find_rsc(ip_0, "httpd-bundle-ip-192.168.122.131", NULL, 0));
	assert_ptr_equal(child_0, native_find_rsc(child_0, "httpd:0", NULL, 0));
	assert_ptr_equal(container_0, native_find_rsc(container_0, "httpd-bundle-docker-0", NULL, 0));
	assert_ptr_equal(remote_0, native_find_rsc(remote_0, "httpd-bundle-0", NULL, 0));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(ip_0, "httpd-bundle-ip-192.168.122.131", cluster01, 0));
	assert_null(native_find_rsc(child_0, "httpd:0", httpd_bundle_0, 0));
	assert_null(native_find_rsc(container_0, "httpd-bundle-docker-0", cluster01, 0));
	assert_null(native_find_rsc(remote_0, "httpd-bundle-0", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(ip_0,
	native_find_rsc(ip_0, "httpd-bundle-ip-192.168.122.131",
	cluster01, pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(ip_0, "httpd-bundle-ip-192.168.122.131",
	cluster02, pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(ip_0, "httpd-bundle-ip-192.168.122.131",
	httpd_bundle_0, pcmk_rsc_match_current_node));
	assert_ptr_equal(child_0,
	native_find_rsc(child_0, "httpd:0", httpd_bundle_0,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd:0", cluster01,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd:0", cluster02,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(container_0,
	native_find_rsc(container_0, "httpd-bundle-docker-0",
	cluster01, pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(container_0, "httpd-bundle-docker-0", cluster02,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(container_0, "httpd-bundle-docker-0",
	httpd_bundle_0, pcmk_rsc_match_current_node));
	assert_ptr_equal(remote_0,
	native_find_rsc(remote_0, "httpd-bundle-0", cluster01,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(remote_0, "httpd-bundle-0", cluster02,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(remote_0, "httpd-bundle-0", httpd_bundle_0,
	pcmk_rsc_match_current_node));

	// Passes because pcmk_rsc_match_basename matches any replica's base name
	assert_ptr_equal(child_0,
	native_find_rsc(child_0, "httpd", NULL,
	pcmk_rsc_match_basename));

	// Passes because pcmk_rsc_match_anon_basename matches
	assert_ptr_equal(child_0,
	native_find_rsc(child_0, "httpd", NULL,
	pcmk_rsc_match_anon_basename));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(child_0,
	native_find_rsc(child_0, "httpd", httpd_bundle_0,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(child_0,
	native_find_rsc(child_0, "httpd", httpd_bundle_0,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd", cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd", cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd", cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(child_0, "httpd", cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));

	/* Fails because incorrect flags were given along with base name. */
	assert_null(native_find_rsc(child_0, "httpd", NULL,
	pcmk_rsc_match_current_node));

	/* And then we check failure possibilities again, except passing httpd-bundle
	* instead of X_0 as the first argument to native_find_rsc.
	*/

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle-ip-192.168.122.131", cluster01, 0));
	assert_null(native_find_rsc(httpd_bundle, "httpd:0", httpd_bundle_0, 0));
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle-docker-0", cluster01, 0));
	assert_null(native_find_rsc(httpd_bundle, "httpd-bundle-0", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(ip_0,
	native_find_rsc(httpd_bundle,
	"httpd-bundle-ip-192.168.122.131",
	cluster01, pcmk_rsc_match_current_node));
	assert_ptr_equal(child_0,
	native_find_rsc(httpd_bundle, "httpd:0", httpd_bundle_0,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(container_0,
	native_find_rsc(httpd_bundle, "httpd-bundle-docker-0",
	cluster01, pcmk_rsc_match_current_node));
	assert_ptr_equal(remote_0,
	native_find_rsc(httpd_bundle, "httpd-bundle-0", cluster01,
	pcmk_rsc_match_current_node));
	return false; // Do not iterate through any further replicas
	}

	static void
	bundle_replica_rsc(void **state)
	{
	pe__foreach_bundle_replica(httpd_bundle, bundle_first_replica, NULL);
	}

	static void
	clone_group_rsc(void **rsc) {
	assert_non_null(mysql_clone_group);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(mysql_clone_group, native_find_rsc(mysql_clone_group, "mysql-clone-group", NULL, 0));
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	NULL, pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	NULL, pcmk_rsc_match_clone_only));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(mysql_clone_group, "mysql-clone-group", cluster01, 0));

	/* Passes because one of mysql-clone-group's children is running on cluster01. */
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	cluster01, pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(mysql_clone_group, "mysql-clone-group", cluster02, 0));

	/* Passes because one of mysql-clone-group's children is running on cluster02. */
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	cluster02, pcmk_rsc_match_current_node));

	// Passes for previous reasons, plus includes pcmk_rsc_match_clone_only
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	cluster01,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_clone_group,
	native_find_rsc(mysql_clone_group, "mysql-clone-group",
	cluster02,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node));
	}

	static void
	clone_group_instance_rsc(void **rsc) {
	pcmk_resource_t *mysql_group_0 = NULL;
	pcmk_resource_t *mysql_group_1 = NULL;

	/* Find the "mysql-group:0" and "mysql-group:1" resources, members of "mysql-clone-group". */
	for (GList *iter = mysql_clone_group->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "mysql-group:0") == 0) {
	mysql_group_0 = rsc;
	} else if (strcmp(rsc->id, "mysql-group:1") == 0) {
	mysql_group_1 = rsc;
	}
	}

	assert_non_null(mysql_group_0);
	assert_non_null(mysql_group_1);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(mysql_group_0, native_find_rsc(mysql_group_0, "mysql-group:0", NULL, 0));
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group:0", NULL,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1, native_find_rsc(mysql_group_1, "mysql-group:1", NULL, 0));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group:1", NULL,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(mysql_group_0, "mysql-group:0", cluster02, 0));
	assert_null(native_find_rsc(mysql_group_1, "mysql-group:1", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group:0", cluster02,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_0, "mysql-group:0", cluster01,
	pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group:1", cluster01,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_1, "mysql-group:1", cluster02,
	pcmk_rsc_match_current_node));

	/* Passes because NULL was passed for node and base name was given, with correct flags. */
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group" , NULL,
	pcmk_rsc_match_clone_only));

	// Passes because pcmk_rsc_match_basename matches any base name
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group" , NULL,
	pcmk_rsc_match_basename));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group" , NULL,
	pcmk_rsc_match_basename));

	// Passes because pcmk_rsc_match_anon_basename matches
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group" , NULL,
	pcmk_rsc_match_anon_basename));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group" , NULL,
	pcmk_rsc_match_anon_basename));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group", cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_group_0, "mysql-group", cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_0, "mysql-group", cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_0, "mysql-group", cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group", cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_group_1, "mysql-group", cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_1, "mysql-group", cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_1, "mysql-group", cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));

	/* Fails because incorrect flags were given along with base name. */
	assert_null(native_find_rsc(mysql_group_0, "mysql-group", NULL,
	pcmk_rsc_match_current_node));
	assert_null(native_find_rsc(mysql_group_1, "mysql-group", NULL,
	pcmk_rsc_match_current_node));

	/* And then we check failure possibilities again, except passing mysql_clone_group
	* instead of mysql_group_X as the first argument to native_find_rsc.
	*/

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(mysql_clone_group, "mysql-group:0", cluster02, 0));
	assert_null(native_find_rsc(mysql_clone_group, "mysql-group:1", cluster01, 0));

	/* Check that the resource is running on the node we expect. */
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_clone_group, "mysql-group:0",
	cluster02, pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_clone_group, "mysql-group",
	cluster02,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_0,
	native_find_rsc(mysql_clone_group, "mysql-group",
	cluster02,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_clone_group, "mysql-group:1",
	cluster01, pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_clone_group, "mysql-group",
	cluster01,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node));
	assert_ptr_equal(mysql_group_1,
	native_find_rsc(mysql_clone_group, "mysql-group",
	cluster01,
	pcmk_rsc_match_anon_basename
	\|pcmk_rsc_match_current_node));
	}

	static void
	clone_group_member_rsc(void **state) {
	pcmk_resource_t *mysql_proxy = NULL;

	/* Find the "mysql-proxy" resource, a member of "mysql-group". */
	for (GList *iter = mysql_clone_group->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "mysql-group:0") == 0) {
	for (GList *iter2 = rsc->children; iter2 != NULL; iter2 = iter2->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter2->data;

	if (strcmp(child->id, "mysql-proxy:0") == 0) {
	mysql_proxy = child;
	break;
	}
	}

	break;
	}
	}

	assert_non_null(mysql_proxy);

	/* Passes because NULL was passed for node, regardless of flags. */
	assert_ptr_equal(mysql_proxy, native_find_rsc(mysql_proxy, "mysql-proxy:0", NULL, 0));
	assert_ptr_equal(mysql_proxy,
	native_find_rsc(mysql_proxy, "mysql-proxy:0", NULL,
	pcmk_rsc_match_current_node));

	/* Passes because resource's parent is a clone. */
	assert_ptr_equal(mysql_proxy,
	native_find_rsc(mysql_proxy, "mysql-proxy:0", NULL,
	pcmk_rsc_match_clone_only));
	assert_ptr_equal(mysql_proxy,
	native_find_rsc(mysql_proxy, "mysql-proxy:0", cluster02,
	pcmk_rsc_match_clone_only
	\|pcmk_rsc_match_current_node));

	/* Fails because mysql-proxy:0 is not running on cluster01, even with the right flags. */
	assert_null(native_find_rsc(mysql_proxy, "mysql-proxy:0", cluster01,
	pcmk_rsc_match_current_node));

	// Fails because pcmk_rsc_match_current_node is required if a node is given
	assert_null(native_find_rsc(mysql_proxy, "mysql-proxy:0", cluster02, 0));

	/* Passes because mysql-proxy:0 is running on cluster02. */
	assert_ptr_equal(mysql_proxy,
	native_find_rsc(mysql_proxy, "mysql-proxy:0", cluster02,
	pcmk_rsc_match_current_node));
	}

	/* TODO: Add tests for finding on assigned node (passing a node without
	* pcmk_rsc_match_current_node, after scheduling, for a resource that is
	* starting/stopping/moving.
	*/
	PCMK__UNIT_TEST(setup, teardown,
	cmocka_unit_test(bad_args),
	cmocka_unit_test(primitive_rsc),
	cmocka_unit_test(group_rsc),
	cmocka_unit_test(inactive_group_rsc),
	cmocka_unit_test(group_member_rsc),
	cmocka_unit_test(inactive_group_member_rsc),
	cmocka_unit_test(clone_rsc),
	cmocka_unit_test(inactive_clone_rsc),
	cmocka_unit_test(clone_instance_rsc),
	cmocka_unit_test(renamed_rsc),
	cmocka_unit_test(bundle_rsc),
	cmocka_unit_test(bundle_replica_rsc),
	cmocka_unit_test(clone_group_rsc),
	cmocka_unit_test(clone_group_instance_rsc),
	cmocka_unit_test(clone_group_member_rsc))
	diff --git a/lib/pengine/tests/native/pe_base_name_eq_test.c b/lib/pengine/tests/native/pe_base_name_eq_test.c
	index cb3c908ed9..f6081ba445 100644
	--- a/lib/pengine/tests/native/pe_base_name_eq_test.c
	+++ b/lib/pengine/tests/native/pe_base_name_eq_test.c
	@@ -1,150 +1,150 @@
	/*
	* Copyright 2022-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>

	#include <crm/common/xml.h>
	#include <crm/common/scheduler.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/status.h>

	xmlNode *input = NULL;
	pcmk_scheduler_t *scheduler = NULL;

	pcmk_resource_t exim_group, promotable_0, promotable_1, dummy;
	pcmk_resource_t httpd_bundle, mysql_group_0, *mysql_group_1;

	static int
	setup(void **state) {
	char *path = NULL;

	crm_xml_init();

	path = crm_strdup_printf("%s/crm_mon.xml", getenv("PCMK_CTS_CLI_DIR"));
	input = filename2xml(path);
	free(path);

	if (input == NULL) {
	return 1;
	}

	scheduler = pe_new_working_set();

	if (scheduler == NULL) {
	return 1;
	}

	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	scheduler->input = input;

	cluster_status(scheduler);

	/* Get references to several resources we use frequently. */
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (strcmp(rsc->id, "dummy") == 0) {
	dummy = rsc;
	} else if (strcmp(rsc->id, "exim-group") == 0) {
	exim_group = rsc;
	} else if (strcmp(rsc->id, "httpd-bundle") == 0) {
	httpd_bundle = rsc;
	} else if (strcmp(rsc->id, "mysql-clone-group") == 0) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	if (strcmp(child->id, "mysql-group:0") == 0) {
	mysql_group_0 = child;
	} else if (strcmp(child->id, "mysql-group:1") == 0) {
	mysql_group_1 = child;
	}
	}
	} else if (strcmp(rsc->id, "promotable-clone") == 0) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	if (strcmp(child->id, "promotable-rsc:0") == 0) {
	promotable_0 = child;
	} else if (strcmp(child->id, "promotable-rsc:1") == 0) {
	promotable_1 = child;
	}
	}
	}
	}

	return 0;
	}

	static int
	teardown(void **state) {
	pe_free_working_set(scheduler);

	return 0;
	}

	static void
	bad_args(void **state) {
	char *id = dummy->id;

	assert_false(pe_base_name_eq(NULL, "dummy"));
	assert_false(pe_base_name_eq(dummy, NULL));

	dummy->id = NULL;
	assert_false(pe_base_name_eq(dummy, "dummy"));
	dummy->id = id;
	}

	static void
	primitive_rsc(void **state) {
	assert_true(pe_base_name_eq(dummy, "dummy"));
	assert_false(pe_base_name_eq(dummy, "DUMMY"));
	assert_false(pe_base_name_eq(dummy, "dUmMy"));
	assert_false(pe_base_name_eq(dummy, "dummy0"));
	assert_false(pe_base_name_eq(dummy, "dummy:0"));
	}

	static void
	group_rsc(void **state) {
	assert_true(pe_base_name_eq(exim_group, "exim-group"));
	assert_false(pe_base_name_eq(exim_group, "EXIM-GROUP"));
	assert_false(pe_base_name_eq(exim_group, "exim-group0"));
	assert_false(pe_base_name_eq(exim_group, "exim-group:0"));
	assert_false(pe_base_name_eq(exim_group, "Public-IP"));
	}

	static void
	clone_rsc(void **state) {
	assert_true(pe_base_name_eq(promotable_0, "promotable-rsc"));
	assert_true(pe_base_name_eq(promotable_1, "promotable-rsc"));

	assert_false(pe_base_name_eq(promotable_0, "promotable-rsc:0"));
	assert_false(pe_base_name_eq(promotable_1, "promotable-rsc:1"));
	assert_false(pe_base_name_eq(promotable_0, "PROMOTABLE-RSC"));
	assert_false(pe_base_name_eq(promotable_1, "PROMOTABLE-RSC"));
	assert_false(pe_base_name_eq(promotable_0, "Promotable-rsc"));
	assert_false(pe_base_name_eq(promotable_1, "Promotable-rsc"));
	}

	static void
	bundle_rsc(void **state) {
	assert_true(pe_base_name_eq(httpd_bundle, "httpd-bundle"));
	assert_false(pe_base_name_eq(httpd_bundle, "HTTPD-BUNDLE"));
	assert_false(pe_base_name_eq(httpd_bundle, "httpd"));
	assert_false(pe_base_name_eq(httpd_bundle, "httpd-docker-0"));
	}

	PCMK__UNIT_TEST(setup, teardown,
	cmocka_unit_test(bad_args),
	cmocka_unit_test(primitive_rsc),
	cmocka_unit_test(group_rsc),
	cmocka_unit_test(clone_rsc),
	cmocka_unit_test(bundle_rsc))
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index f15abd399c..8f641826ed 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5131 +1,5131 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	- * use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
	+ * use pcmk__set_scheduler_flags()/pcmk__clear_scheduler_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	static gboolean
	is_dangling_guest_node(pcmk_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider
	* \c PCMK_OPT_PRIORITY_FENCING_DELAY
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pe__node_name(node), reason, rsc->id);
	} else {
	pcmk__sched_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pe__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pe__node_name(node), reason);
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pe__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	pcmk__sched_warn("Remote node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply PCMK_OPT_PRIORITY_FENCING_DELAY for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	pcmk__sched_warn("Cluster node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" XML_NVPAIR_ATTR_NAME "='" PCMK_META_REQUIRES "') " \
	"and @" XML_NVPAIR_ATTR_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	- pe__set_working_set_flags(scheduler, flag);
	+ pcmk__set_scheduler_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	guint interval_ms = 0U;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, scheduler);

	verify_pe_options(scheduler->config_hash);

	set_config_flag(scheduler, PCMK_OPT_ENABLE_STARTUP_PROBES,
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT
	" is nonzero");
	- pe__set_working_set_flags(scheduler, pcmk_sched_have_fencing);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
	pcmk_parse_interval_spec(value, &interval_ms);

	if (interval_ms >= INT_MAX) {
	scheduler->stonith_timeout = INT_MAX;
	} else {
	scheduler->stonith_timeout = (int) interval_ms;
	}
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, PCMK_OPT_STONITH_ENABLED,
	pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pe_pref(scheduler->config_hash,
	PCMK_OPT_STONITH_ACTION);
	if (!strcmp(scheduler->stonith_action, "poweroff")) {
	pcmk__warn_once(pcmk__wo_poweroff,
	"Support for " PCMK_OPT_STONITH_ACTION " of "
	"'poweroff' is deprecated and will be removed in a "
	"future release (use 'off' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, PCMK_OPT_CONCURRENT_FENCING,
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(scheduler->config_hash,
	PCMK_OPT_PRIORITY_FENCING_DELAY);
	if (value) {
	pcmk_parse_interval_spec(value, &interval_ms);
	scheduler->priority_fencing_delay = (int) (interval_ms / 1000);
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ALL_RESOURCES,
	pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)));

	set_config_flag(scheduler, PCMK_OPT_SYMMETRIC_CLUSTER,
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NO_QUORUM_POLICY);

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, PCMK_XA_NO_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop' because fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_RESOURCES,
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_ACTIONS,
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(scheduler->config_hash, PCMK__OPT_REMOVE_AFTER_STOP);
	if (value != NULL) {
	if (crm_is_true(value)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_remove_after_stop);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pcmk__warn_once(pcmk__wo_remove_after,
	"Support for the " PCMK__OPT_REMOVE_AFTER_STOP
	" cluster property is deprecated and will be "
	"removed in a future release");
	#endif
	} else {
	- pe__clear_working_set_flags(scheduler,
	+ pcmk__clear_scheduler_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, PCMK_OPT_MAINTENANCE_MODE,
	pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags,
	pcmk_sched_in_maintenance)));

	set_config_flag(scheduler, PCMK_OPT_START_FAILURE_IS_FATAL,
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, PCMK_OPT_STARTUP_FENCING,
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pcmk__warn_once(pcmk__wo_blind,
	"Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy = pe_pref(scheduler->config_hash,
	PCMK_OPT_PLACEMENT_STRATEGY);
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, PCMK_OPT_SHUTDOWN_LOCK,
	pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(scheduler->config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &(scheduler->shutdown_lock));
	scheduler->shutdown_lock /= 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &(scheduler->node_pending_timeout));
	scheduler->node_pending_timeout /= 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pe_find_node(scheduler->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = scheduler;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	- pe__set_working_set_flags(scheduler, pcmk_sched_have_remote_nodes);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pcmk__warn_once(pcmk__wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (name == NULL) { // Sanity
	continue;
	}

	if (strcmp(name, PCMK_META_REMOTE_NODE) == 0) {
	remote_name = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ADDR) == 0) {
	remote_server = value;

	} else if (strcmp(name, PCMK_META_REMOTE_PORT) == 0) {
	remote_port = value;

	} else if (strcmp(name, PCMK_META_REMOTE_CONNECT_TIMEOUT) == 0) {
	connect_timeout = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ALLOW_MIGRATE) == 0) {
	remote_allow_migrate = value;

	} else if (strcmp(name, PCMK_META_IS_MANAGED) == 0) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pe_find_node(scheduler->nodes, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	setup_container(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, scheduler);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, PCMK__META_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pcmk_resource_t *container = pe_find_resource(scheduler->resources,
	container_id);

	if (container) {
	rsc->container = container;
	pe__set_resource_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's container is %s",
	rsc->id, container_id);
	} else {
	pcmk__config_err("Resource %s: Unknown resource container (%s)",
	rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(scheduler->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pe__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(scheduler->template_rsc_sets, strdup(id),
	NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " PCMK_XA_ID "='%s'>", xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	setup_container(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the "
	PCMK_OPT_STONITH_ENABLED " option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " PCMK_XA_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " PCMK_XA_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, PCMK_XA_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pe__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pe__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pe__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pe__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pe__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY " attribute "
	"for %s because disabling resource discovery is "
	"not allowed for cluster nodes", pe__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring invalid " XML_CIB_TAG_STATE " entry without "
	PCMK_XA_ID);
	crm_log_xml_info(state, "missing-id");
	return;
	}

	uname = crm_element_value(state, XML_ATTR_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created node_state entry doesn't have an uname
	* yet. We should recognize the node as `pending` and wait for it to
	* join CPG.
	*/
	crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	XML_ATTR_UNAME, id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pe__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, XML_ATTR_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	* \param[in] node_state Node's node_state entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for PCMK_OPT_NODE_PENDING_TIMEOUT.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	" in " XML_CIB_TAG_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	* \param[in] node_state Node's node_state entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK__XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, OFFLINESTATUS,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, ONLINESTATUS, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_CRMD " in "
	XML_CIB_TAG_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	* \param[in] node_state Node's node_state entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pe_node_attribute_raw(node, PCMK_NODE_ATTR_TERMINATE);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pe__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pe__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pe__node_name(this_node), join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* \c PCMK_OPT_NODE_PENDING_TIMEOUT, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->details->data_set) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK__XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK__XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK__XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pe__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pe__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pe__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider PCMK_OPT_PRIORITY_FENCING_DELAY for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pe__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->details->remote_rsc;
	pcmk_resource_t *container = NULL;
	pcmk_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pe__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pe__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pe__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pe__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(scheduler->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, PCMK__META_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pe__set_resource_flags(rsc, pcmk_rsc_removed_filler);
	}
	pe__set_resource_flags(rsc, pcmk_rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);

	// find_rsc() because we might be a cloned group
	pcmk_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pe__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of \c PCMK_META_CLONE_MAX
	* instances); (3) a newly created orphan (that is, \c PCMK_META_CLONE_MAX
	* instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pe__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if
	* PCMK_META_GLOBALLY_UNIQUE was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pcmk_node_t *) locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if PCMK_META_GLOBALLY_UNIQUE is switched from true
	* to false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pe__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when PCMK_META_CLONE_MAX=0,
	* we create a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pe__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__",
	scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pe__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pe__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pe__clear_resource_flags(rsc, pcmk_rsc_managed);
	pe__set_resource_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pe__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pe__set_resource_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pe__set_resource_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pe__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pe__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pe__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pe__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pe__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pe__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, PCMK_OPT_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pe__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pcmk_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring invalid " XML_LRM_TAG_RESOURCE
	" entry: No " PCMK_XA_ID);
	crm_log_xml_info(lrm_resource, "missing-id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pe__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, PCMK_META_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, PCMK__META_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(scheduler->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, XML_LRM_ATTR_TASK);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	pcmk__config_err("Ignoring resource history entry %s without "
	XML_LRM_ATTR_MIGRATE_SOURCE " and "
	XML_LRM_ATTR_MIGRATE_TARGET, ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	ID(entry), *source_name, pe__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	ID(entry), *target_name, pe__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pe__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
	&from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pe__set_resource_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pe__clear_resource_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pe__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pe__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -INFINITY;
	pcmk_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	pcmk__sched_warn("Unexpected result (%s%s%s) was recorded for %s of "
	"%s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	history->rsc->id, pe__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	fail2text(*on_fail), fail2text(config_on_fail),
	history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(config_on_fail), role2text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pe__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pe__clear_resource_flags(history->rsc, pcmk_rsc_managed);
	pe__set_resource_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	pcmk__config_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pe__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pe__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pe__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pe__xe_history_key(xml_op),
	node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pe_find_node(scheduler->nodes, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pe__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	/* Resource has a PCMK_META_FAILURE_TIMEOUT and history entry has a
	* timestamp
	*/

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pe__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pe__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pe__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	history->id, history->rsc->id,
	pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	XML_LRM_ATTR_OPSTATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_RC " '%s'",
	history->id, history->rsc->id,
	pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	XML_LRM_ATTR_RC),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml,
	XML_LRM_ATTR_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pe__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, XML_LRM_ATTR_TASK),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	XML_RSC_OP_LAST_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pe__set_resource_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	XML_LRM_ATTR_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	pcmk__config_err("Ignoring resource history entry for %s on %s "
	"without ID", rsc->id, pe__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (history.task == NULL) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"without " XML_LRM_ATTR_TASK,
	history.id, rsc->id, pe__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
	&(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pe__node_name(node), history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pe__xe_history_key(xml_op);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pe__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pe__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pe__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pe__set_resource_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
	last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	pe__set_resource_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pe__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	pcmk__sched_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " %s",
	parent->id,
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pe__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_CLUSTER_NAME);
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, scheduler);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(scheduler->input, XML_CIB_TAG_STATUS, TRUE);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(scheduler->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc,
	PCMK_XA_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index afb71cc299..de3766d5f8 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,2086 +1,2086 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/lrmd_internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/ipc_attrd_internal.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <pacemaker-internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/cib/internal.h>

	#define SUMMARY "crm_resource - perform tasks related to Pacemaker cluster resources"

	enum rsc_command {
	cmd_none = 0, // No command option given (yet)
	cmd_ban,
	cmd_cleanup,
	cmd_clear,
	cmd_colocations,
	cmd_cts,
	cmd_delete,
	cmd_delete_param,
	cmd_digests,
	cmd_execute_agent,
	cmd_fail,
	cmd_get_param,
	cmd_get_property,
	cmd_list_active_ops,
	cmd_list_agents,
	cmd_list_all_ops,
	cmd_list_alternatives,
	cmd_list_instances,
	cmd_list_providers,
	cmd_list_resources,
	cmd_list_standards,
	cmd_locate,
	cmd_metadata,
	cmd_move,
	cmd_query_raw_xml,
	cmd_query_xml,
	cmd_refresh,
	cmd_restart,
	cmd_set_param,
	cmd_set_property,
	cmd_wait,
	cmd_why,
	};

	struct {
	enum rsc_command rsc_cmd; // crm_resource command to perform

	// Infrastructure that given command needs to work
	gboolean require_cib; // Whether command requires CIB IPC
	int cib_options; // Options to use with CIB IPC calls
	gboolean require_crmd; // Whether command requires controller IPC
	gboolean require_scheduler; // Whether command requires scheduler data
	gboolean require_resource; // Whether command requires resource specified
	gboolean require_node; // Whether command requires node specified
	int find_flags; // Flags to use when searching for resource

	// Command-line option values
	gchar *rsc_id; // Value of --resource
	gchar *rsc_type; // Value of --resource-type
	gboolean force; // --force was given
	gboolean clear_expired; // --expired was given
	gboolean recursive; // --recursive was given
	gboolean promoted_role_only; // --promoted was given
	gchar *host_uname; // Value of --node
	gchar *interval_spec; // Value of --interval
	gchar *move_lifetime; // Value of --lifetime
	gchar *operation; // Value of --operation
	const char *attr_set_type; // Instance, meta, utilization, or element attribute
	gchar *prop_id; // --nvpair (attribute XML ID)
	char *prop_name; // Attribute name
	gchar *prop_set; // --set-name (attribute block XML ID)
	gchar *prop_value; // --parameter-value (attribute value)
	int timeout_ms; // Parsed from --timeout value
	char *agent_spec; // Standard and/or provider and/or agent
	gchar *xml_file; // Value of (deprecated) --xml-file
	int check_level; // Optional value of --validate or --force-check

	// Resource configuration specified via command-line arguments
	gboolean cmdline_config; // Resource configuration was via arguments
	char *v_agent; // Value of --agent
	char *v_class; // Value of --class
	char *v_provider; // Value of --provider
	GHashTable *cmdline_params; // Resource parameters specified

	// Positional command-line arguments
	gchar **remainder; // Positional arguments as given
	GHashTable *override_params; // Resource parameter values that override config
	} options = {
	.attr_set_type = XML_TAG_ATTR_SETS,
	.check_level = -1,
	.cib_options = cib_sync_call,
	.require_cib = TRUE,
	.require_scheduler = TRUE,
	.require_resource = TRUE,
	};

	#if 0
	// @COMPAT @TODO enable this at next backward compatibility break
	#define SET_COMMAND(cmd) do { \
	if (options.rsc_cmd != cmd_none) { \
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE, \
	"Only one command option may be specified"); \
	return FALSE; \
	} \
	options.rsc_cmd = (cmd); \
	} while (0)
	#else
	#define SET_COMMAND(cmd) do { \
	if (options.rsc_cmd != cmd_none) { \
	reset_options(); \
	} \
	options.rsc_cmd = (cmd); \
	} while (0)
	#endif

	gboolean agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_agents_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_providers_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_standards_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean metadata_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean option_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean restart_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean digests_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error);
	gboolean wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static crm_exit_t exit_code = CRM_EX_OK;
	static pcmk__output_t *out = NULL;
	static pcmk__common_args_t *args = NULL;

	// Things that should be cleaned up on exit
	static GError *error = NULL;
	static GMainLoop *mainloop = NULL;
	static cib_t *cib_conn = NULL;
	static pcmk_ipc_api_t *controld_api = NULL;
	static pcmk_scheduler_t *scheduler = NULL;

	#define MESSAGE_TIMEOUT_S 60

	#define INDENT " "

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	// Clean up and exit
	static crm_exit_t
	bye(crm_exit_t ec)
	{
	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, ec, true, NULL);
	pcmk__output_free(out);
	}
	pcmk__unregister_formats();

	if (cib_conn != NULL) {
	cib_t *save_cib_conn = cib_conn;

	cib_conn = NULL; // Ensure we can't free this twice
	cib__clean_up_connection(&save_cib_conn);
	}

	if (controld_api != NULL) {
	pcmk_ipc_api_t *save_controld_api = controld_api;

	controld_api = NULL; // Ensure we can't free this twice
	pcmk_free_ipc_api(save_controld_api);
	}

	if (mainloop != NULL) {
	g_main_loop_unref(mainloop);
	mainloop = NULL;
	}

	pe_free_working_set(scheduler);
	scheduler = NULL;
	crm_exit(ec);
	return ec;
	}

	static void
	quit_main_loop(crm_exit_t ec)
	{
	exit_code = ec;
	if (mainloop != NULL) {
	GMainLoop *mloop = mainloop;

	mainloop = NULL; // Don't re-enter this block
	pcmk_quit_main_loop(mloop, 10);
	g_main_loop_unref(mloop);
	}
	}

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	// Start with newline because "Waiting for ..." message doesn't have one
	if (error != NULL) {
	g_clear_error(&error);
	}

	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_TIMEOUT,
	_("Aborting because no messages received in %d seconds"), MESSAGE_TIMEOUT_S);

	quit_main_loop(CRM_EX_TIMEOUT);
	return FALSE;
	}

	static void
	controller_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void event_data, void user_data)
	{
	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
	crm_info("Connection to controller was terminated");
	}
	quit_main_loop(exit_code);
	break;

	case pcmk_ipc_event_reply:
	if (status != CRM_EX_OK) {
	out->err(out, "Error: bad reply from controller: %s",
	crm_exit_str(status));
	pcmk_disconnect_ipc(api);
	quit_main_loop(status);
	} else {
	if ((pcmk_controld_api_replies_expected(api) == 0)
	&& mainloop && g_main_loop_is_running(mainloop)) {
	out->info(out, "... got reply (done)");
	crm_debug("Got all the replies we expected");
	pcmk_disconnect_ipc(api);
	quit_main_loop(CRM_EX_OK);
	} else {
	out->info(out, "... got reply");
	}
	}
	break;

	default:
	break;
	}
	}

	static void
	start_mainloop(pcmk_ipc_api_t *capi)
	{
	unsigned int count = pcmk_controld_api_replies_expected(capi);

	if (count > 0) {
	out->info(out, "Waiting for %u %s from the controller",
	count, pcmk__plural_alt(count, "reply", "replies"));
	exit_code = CRM_EX_DISCONNECT; // For unexpected disconnects
	mainloop = g_main_loop_new(NULL, FALSE);
	g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
	g_main_loop_run(mainloop);
	}
	}

	static int
	compare_id(gconstpointer a, gconstpointer b)
	{
	return strcmp((const char )a, (const char )b);
	}

	static GList *
	build_constraint_list(xmlNode *root)
	{
	GList *retval = NULL;
	xmlNode *cib_constraints = NULL;
	xmlXPathObjectPtr xpathObj = NULL;
	int ndx = 0;

	cib_constraints = pcmk_find_cib_element(root, XML_CIB_TAG_CONSTRAINTS);
	xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);

	for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
	xmlNode *match = getXpathResult(xpathObj, ndx);
	retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
	}

	freeXpathObject(xpathObj);
	return retval;
	}

	/* short option letters still available: eEJkKXyYZ */

	static GOptionEntry query_entries[] = {
	{ "list", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all cluster resources with status",
	NULL },
	{ "list-raw", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List IDs of all instantiated resources (individual members\n"
	INDENT "rather than groups etc.)",
	NULL },
	{ "list-cts", 'c', G_OPTION_FLAG_HIDDEN\|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	NULL,
	NULL },
	{ "list-operations", 'O', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List active resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-all-operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, list_cb,
	"List all resource operations, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },
	{ "list-standards", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_standards_cb,
	"List supported standards",
	NULL },
	{ "list-ocf-providers", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	list_providers_cb,
	"List all available OCF providers",
	NULL },
	{ "list-agents", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_agents_cb,
	"List all agents available for the named standard and/or provider",
	"STD:PROV" },
	{ "list-ocf-alternatives", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	list_alternatives_cb,
	"List all available providers for the named OCF agent",
	"AGENT" },
	{ "show-metadata", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK,
	metadata_cb,
	"Show the metadata for the named class:provider:agent",
	"SPEC" },
	{ "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (after any template expansion)",
	NULL },
	{ "query-xml-raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show XML configuration of resource (before any template expansion)",
	NULL },
	{ "get-parameter", 'g', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named parameter for resource (use instance attribute\n"
	INDENT "unless --element, --meta, or --utilization is specified)",
	"PARAM" },
	{ "get-property", 'G', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, get_param_prop_cb,
	"Display named property of resource ('class', 'type', or 'provider') "
	"(requires --resource)",
	"PROPERTY" },
	{ "locate", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Show node(s) currently running resource",
	NULL },
	{ "constraints", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Display the location and colocation constraints that apply to a\n"
	INDENT "resource, and if --recursive is specified, to the resources\n"
	INDENT "directly or indirectly involved in those colocations.\n"
	INDENT "If the named resource is part of a group, or a clone or\n"
	INDENT "bundle instance, constraints for the collective resource\n"
	INDENT "will be shown unless --force is given.",
	NULL },
	{ "stack", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Equivalent to --constraints --recursive",
	NULL },
	{ "why", 'Y', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, why_cb,
	"Show why resources are not running, optionally filtered by\n"
	INDENT "--resource and/or --node",
	NULL },

	{ NULL }
	};

	static GOptionEntry command_entries[] = {
	{ "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"Validate resource configuration by calling agent's validate-all\n"
	INDENT "action. The configuration may be specified either by giving an\n"
	INDENT "existing resource name with -r, or by specifying --class,\n"
	INDENT "--agent, and --provider arguments, along with any number of\n"
	INDENT "--option arguments. An optional LEVEL argument can be given\n"
	INDENT "to control the level of checking performed.",
	"LEVEL" },
	{ "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"If resource has any past failures, clear its history and fail\n"
	INDENT "count. Optionally filtered by --resource, --node, --operation\n"
	INDENT "and --interval (otherwise all). --operation and --interval\n"
	INDENT "apply to fail counts, but entire history is always clear, to\n"
	INDENT "allow current state to be rechecked. If the named resource is\n"
	INDENT "part of a group, or one numbered instance of a clone or bundled\n"
	INDENT "resource, the clean-up applies to the whole collective resource\n"
	INDENT "unless --force is given.",
	NULL },
	{ "refresh", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, cleanup_refresh_cb,
	"Delete resource's history (including failures) so its current state\n"
	INDENT "is rechecked. Optionally filtered by --resource and --node\n"
	INDENT "(otherwise all). If the named resource is part of a group, or one\n"
	INDENT "numbered instance of a clone or bundled resource, the refresh\n"
	INDENT "applies to the whole collective resource unless --force is given.",
	NULL },
	{ "set-parameter", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Set named parameter for resource (requires -v). Use instance\n"
	INDENT "attribute unless --element, --meta, or --utilization is "
	"specified.",
	"PARAM" },
	{ "delete-parameter", 'd', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, set_delete_param_cb,
	"Delete named parameter for resource. Use instance attribute\n"
	INDENT "unless --element, --meta or, --utilization is specified.",
	"PARAM" },
	{ "set-property", 'S', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, set_prop_cb,
	"Set named property of resource ('class', 'type', or 'provider') "
	"(requires -r, -t, -v)",
	"PROPERTY" },

	{ NULL }
	};

	static GOptionEntry location_entries[] = {
	{ "move", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to move resource. If --node is specified,\n"
	INDENT "the constraint will be to move to that node, otherwise it\n"
	INDENT "will be to ban the current node. Unless --force is specified\n"
	INDENT "this will return an error if the resource is already running\n"
	INDENT "on the specified node. If --force is specified, this will\n"
	INDENT "always ban the current node.\n"
	INDENT "Optional: --lifetime, --promoted. NOTE: This may prevent the\n"
	INDENT "resource from running on its previous location until the\n"
	INDENT "implicit constraint expires or is removed with --clear.",
	NULL },
	{ "ban", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Create a constraint to keep resource off a node.\n"
	INDENT "Optional: --node, --lifetime, --promoted.\n"
	INDENT "NOTE: This will prevent the resource from running on the\n"
	INDENT "affected node until the implicit constraint expires or is\n"
	INDENT "removed with --clear. If --node is not specified, it defaults\n"
	INDENT "to the node currently running the resource for primitives\n"
	INDENT "and groups, or the promoted instance of promotable clones with\n"
	INDENT PCMK_META_PROMOTED_MAX "=1 (all other situations result in an\n"
	INDENT "error as there is no sane default).",
	NULL },
	{ "clear", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, flag_cb,
	"Remove all constraints created by the --ban and/or --move\n"
	INDENT "commands. Requires: --resource. Optional: --node, --promoted,\n"
	INDENT "--expired. If --node is not specified, all constraints created\n"
	INDENT "by --ban and --move will be removed for the named resource. If\n"
	INDENT "--node and --force are specified, any constraint created by\n"
	INDENT "--move will be cleared, even if it is not for the specified\n"
	INDENT "node. If --expired is specified, only those constraints whose\n"
	INDENT "lifetimes have expired will be removed.",
	NULL },
	{ "expired", 'e', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, expired_cb,
	"Modifies the --clear argument to remove constraints with\n"
	INDENT "expired lifetimes.",
	NULL },
	{ "lifetime", 'u', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.move_lifetime,
	"Lifespan (as ISO 8601 duration) of created constraints (with\n"
	INDENT "-B, -M) see https://en.wikipedia.org/wiki/ISO_8601#Durations)",
	"TIMESPEC" },
	{ "promoted", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.promoted_role_only,
	"Limit scope of command to promoted role (with -B, -M, -U). For\n"
	INDENT "-B and -M, previously promoted instances may remain\n"
	INDENT "active in the unpromoted role.",
	NULL },

	// Deprecated since 2.1.0
	{ "master", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.promoted_role_only,
	"Deprecated: Use --promoted instead", NULL },

	{ NULL }
	};

	static GOptionEntry advanced_entries[] = {
	{ "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, delete_cb,
	"(Advanced) Delete a resource from the CIB. Required: -t",
	NULL },
	{ "fail", 'F', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, fail_cb,
	"(Advanced) Tell the cluster this resource has failed",
	NULL },
	{ "restart", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, restart_cb,
	"(Advanced) Tell the cluster to restart this resource and\n"
	INDENT "anything that depends on it",
	NULL },
	{ "wait", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, wait_cb,
	"(Advanced) Wait until the cluster settles into a stable state",
	NULL },
	{ "digests", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, digests_cb,
	"(Advanced) Show parameter hashes that Pacemaker uses to detect\n"
	INDENT "configuration changes (only accurate if there is resource\n"
	INDENT "history on the specified node). Required: --resource, --node.\n"
	INDENT "Optional: any NAME=VALUE parameters will be used to override\n"
	INDENT "the configuration (to see what the hash would be with those\n"
	INDENT "changes).",
	NULL },
	{ "force-demote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and demote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-stop", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and stop a resource on the local node",
	NULL },
	{ "force-start", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and start a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-promote", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and promote a resource on the local\n"
	INDENT "node. Unless --force is specified, this will refuse to do so if\n"
	INDENT "the cluster believes the resource is a clone instance already\n"
	INDENT "running on the local node.",
	NULL },
	{ "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	validate_or_force_cb,
	"(Advanced) Bypass the cluster and check the state of a resource on\n"
	INDENT "the local node. An optional LEVEL argument can be given\n"
	INDENT "to control the level of checking performed.",
	"LEVEL" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.host_uname,
	"Node name",
	"NAME" },
	{ "recursive", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.recursive,
	"Follow colocation chains when using --set-parameter or --constraints",
	NULL },
	{ "resource-type", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_type,
	"Resource XML element (primitive, group, etc.) (with -D)",
	"ELEMENT" },
	{ "parameter-value", 'v', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_value,
	"Value to use with -p",
	"PARAM" },
	{ "meta", 'm', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource meta-attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "utilization", 'z', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource utilization attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "element", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attr_set_type_cb,
	"Use resource element attribute instead of instance attribute\n"
	INDENT "(with -p, -g, -d)",
	NULL },
	{ "operation", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.operation,
	"Operation to clear instead of all (with -C -r)",
	"OPERATION" },
	{ "interval", 'I', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.interval_spec,
	"Interval of operation to clear (default 0) (with -C -r -n)",
	"N" },
	{ "class", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, class_cb,
	"The standard the resource agent conforms to (for example, ocf).\n"
	INDENT "Use with --agent, --provider, --option, and --validate.",
	"CLASS" },
	{ "agent", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The agent to use (for example, IPaddr). Use with --class,\n"
	INDENT "--provider, --option, and --validate.",
	"AGENT" },
	{ "provider", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, agent_provider_cb,
	"The vendor that supplies the resource agent (for example,\n"
	INDENT "heartbeat). Use with --class, --agent, --option, and --validate.",
	"PROVIDER" },
	{ "option", 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, option_cb,
	"Specify a device configuration parameter as NAME=VALUE (may be\n"
	INDENT "specified multiple times). Use with --validate and without the\n"
	INDENT "-r option.",
	"PARAM" },
	{ "set-name", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_set,
	"(Advanced) XML ID of attributes element to use (with -p, -d)",
	"ID" },
	{ "nvpair", 'i', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.prop_id,
	"(Advanced) XML ID of nvpair element to use (with -p, -d)",
	"ID" },
	{ "timeout", 'T', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, timeout_cb,
	"(Advanced) Abort if command does not finish in this time (with\n"
	INDENT "--restart, --wait, --force-*)",
	"N" },
	{ "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
	"Force the action to be performed. See help for individual commands for\n"
	INDENT "additional behavior.",
	NULL },
	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_FILENAME, &options.xml_file,
	NULL,
	"FILE" },
	{ "host-uname", 'H', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.host_uname,
	NULL,
	"HOST" },

	{ NULL }
	};

	static void
	reset_options(void) {
	options.require_crmd = FALSE;
	options.require_node = FALSE;

	options.require_cib = TRUE;
	options.require_scheduler = TRUE;
	options.require_resource = TRUE;

	options.find_flags = 0;
	}

	gboolean
	agent_provider_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.cmdline_config = TRUE;
	options.require_resource = FALSE;

	if (pcmk__str_eq(option_name, "--provider", pcmk__str_casei)) {
	pcmk__str_update(&options.v_provider, optarg);
	} else {
	pcmk__str_update(&options.v_agent, optarg);
	}

	return TRUE;
	}

	gboolean
	attr_set_type_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-m", "--meta", NULL)) {
	options.attr_set_type = XML_TAG_META_SETS;
	} else if (pcmk__str_any_of(option_name, "-z", "--utilization", NULL)) {
	options.attr_set_type = XML_TAG_UTILIZATION;
	} else if (pcmk__str_eq(option_name, "--element", pcmk__str_casei)) {
	options.attr_set_type = ATTR_SET_ELEMENT;
	}
	return TRUE;
	}

	gboolean
	class_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.v_class, optarg);
	options.cmdline_config = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	cleanup_refresh_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-C", "--cleanup", NULL)) {
	SET_COMMAND(cmd_cleanup);
	} else {
	SET_COMMAND(cmd_refresh);
	}

	options.require_resource = FALSE;
	if (getenv("CIB_file") == NULL) {
	options.require_crmd = TRUE;
	}
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_anon_basename;
	return TRUE;
	}

	gboolean
	delete_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_delete);
	options.require_scheduler = FALSE;
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	return TRUE;
	}

	gboolean
	expired_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.clear_expired = TRUE;
	options.require_resource = FALSE;
	return TRUE;
	}

	static void
	get_agent_spec(const gchar *optarg)
	{
	options.require_cib = FALSE;
	options.require_scheduler = FALSE;
	options.require_resource = FALSE;
	pcmk__str_update(&options.agent_spec, optarg);
	}

	gboolean
	list_agents_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_agents);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_providers_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_providers);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	list_standards_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_list_standards);
	options.require_cib = FALSE;
	options.require_scheduler = FALSE;
	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	list_alternatives_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_list_alternatives);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	metadata_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_metadata);
	get_agent_spec(optarg);
	return TRUE;
	}

	gboolean
	option_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	char *name = NULL;
	char *value = NULL;

	if (pcmk__scan_nvpair(optarg, &name, &value) != 2) {
	return FALSE;
	}
	if (options.cmdline_params == NULL) {
	options.cmdline_params = pcmk__strkey_table(free, free);
	}
	g_hash_table_replace(options.cmdline_params, name, value);
	return TRUE;
	}

	gboolean
	fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_fail);
	options.require_crmd = TRUE;
	options.require_node = TRUE;
	return TRUE;
	}

	gboolean
	flag_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-U", "--clear", NULL)) {
	SET_COMMAND(cmd_clear);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;
	} else if (pcmk__str_any_of(option_name, "-B", "--ban", NULL)) {
	SET_COMMAND(cmd_ban);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;
	} else if (pcmk__str_any_of(option_name, "-M", "--move", NULL)) {
	SET_COMMAND(cmd_move);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;
	} else if (pcmk__str_any_of(option_name, "-q", "--query-xml", NULL)) {
	SET_COMMAND(cmd_query_xml);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	} else if (pcmk__str_any_of(option_name, "-w", "--query-xml-raw", NULL)) {
	SET_COMMAND(cmd_query_raw_xml);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	} else if (pcmk__str_any_of(option_name, "-W", "--locate", NULL)) {
	SET_COMMAND(cmd_locate);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;

	} else if (pcmk__str_any_of(option_name, "-a", "--constraints", NULL)) {
	SET_COMMAND(cmd_colocations);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;

	} else if (pcmk__str_any_of(option_name, "-A", "--stack", NULL)) {
	SET_COMMAND(cmd_colocations);
	options.find_flags = pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename;
	options.recursive = TRUE;
	}

	return TRUE;
	}

	gboolean
	get_param_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-g", "--get-parameter", NULL)) {
	SET_COMMAND(cmd_get_param);
	} else {
	SET_COMMAND(cmd_get_property);
	}

	pcmk__str_update(&options.prop_name, optarg);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	return TRUE;
	}

	gboolean
	list_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-c", "--list-cts", NULL)) {
	SET_COMMAND(cmd_cts);
	} else if (pcmk__str_any_of(option_name, "-L", "--list", NULL)) {
	SET_COMMAND(cmd_list_resources);
	} else if (pcmk__str_any_of(option_name, "-l", "--list-raw", NULL)) {
	SET_COMMAND(cmd_list_instances);
	} else if (pcmk__str_any_of(option_name, "-O", "--list-operations", NULL)) {
	SET_COMMAND(cmd_list_active_ops);
	} else {
	SET_COMMAND(cmd_list_all_ops);
	}

	options.require_resource = FALSE;
	return TRUE;
	}

	gboolean
	set_delete_param_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_any_of(option_name, "-p", "--set-parameter", NULL)) {
	SET_COMMAND(cmd_set_param);
	} else {
	SET_COMMAND(cmd_delete_param);
	}

	pcmk__str_update(&options.prop_name, optarg);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	return TRUE;
	}

	gboolean
	set_prop_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_set_property);
	options.require_scheduler = FALSE;
	pcmk__str_update(&options.prop_name, optarg);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_basename;
	return TRUE;
	}

	gboolean
	timeout_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.timeout_ms = crm_get_msec(optarg);
	return TRUE;
	}

	gboolean
	validate_or_force_cb(const gchar option_name, const gchar optarg,
	gpointer data, GError **error)
	{
	SET_COMMAND(cmd_execute_agent);
	if (options.operation) {
	g_free(options.operation);
	}
	options.operation = g_strdup(option_name + 2); // skip "--"
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_anon_basename;
	if (options.override_params == NULL) {
	options.override_params = pcmk__strkey_table(free, free);
	}

	if (optarg != NULL) {
	if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
	g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
	_("Invalid check level setting: %s"), optarg);
	return FALSE;
	}
	}

	return TRUE;
	}

	gboolean
	restart_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_restart);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_anon_basename;
	return TRUE;
	}

	gboolean
	digests_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	SET_COMMAND(cmd_digests);
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_anon_basename;
	if (options.override_params == NULL) {
	options.override_params = pcmk__strkey_table(free, free);
	}
	options.require_node = TRUE;
	options.require_scheduler = TRUE;
	return TRUE;
	}

	gboolean
	wait_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_wait);
	options.require_resource = FALSE;
	options.require_scheduler = FALSE;
	return TRUE;
	}

	gboolean
	why_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	SET_COMMAND(cmd_why);
	options.require_resource = FALSE;
	options.find_flags = pcmk_rsc_match_history\|pcmk_rsc_match_anon_basename;
	return TRUE;
	}

	static int
	ban_or_move(pcmk__output_t out, pcmk_resource_t rsc,
	const char *move_lifetime)
	{
	int rc = pcmk_rc_ok;
	pcmk_node_t *current = NULL;
	unsigned int nactive = 0;

	CRM_CHECK(rsc != NULL, return EINVAL);

	current = pe__find_active_requires(rsc, &nactive);

	if (nactive == 1) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime,
	cib_conn, options.cib_options, options.promoted_role_only,
	PCMK__ROLE_PROMOTED);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	int count = 0;
	GList *iter = NULL;

	current = NULL;
	for(iter = rsc->children; iter; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == pcmk_role_promoted) {
	count++;
	current = pe__current_node(child);
	}
	}

	if(count == 1 && current) {
	rc = cli_resource_ban(out, options.rsc_id, current->details->uname, move_lifetime,
	cib_conn, options.cib_options, options.promoted_role_only,
	PCMK__ROLE_PROMOTED);

	} else {
	rc = EINVAL;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("Resource '%s' not moved: active in %d locations (promoted in %d).\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node."
	"To prevent '%s' from being promoted at a specific "
	"location, specify a node and the --promoted option."),
	options.rsc_id, nactive, count, options.rsc_id, options.rsc_id);
	}

	} else {
	rc = EINVAL;
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("Resource '%s' not moved: active in %d locations.\n"
	"To prevent '%s' from running on a specific location, "
	"specify a node."),
	options.rsc_id, nactive, options.rsc_id);
	}

	return rc;
	}

	static void
	cleanup(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Erasing failures of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(controld_api, options.host_uname, rsc,
	options.operation, options.interval_spec, TRUE,
	scheduler, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, rsc, node);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	clear_constraints(pcmk__output_t out, xmlNodePtr cib_xml_copy)
	{
	GList *before = NULL;
	GList *after = NULL;
	GList *remaining = NULL;
	GList *ele = NULL;
	pcmk_node_t *dest = NULL;
	int rc = pcmk_rc_ok;

	if (!out->is_quiet(out)) {
	before = build_constraint_list(scheduler->input);
	}

	if (options.clear_expired) {
	rc = cli_resource_clear_all_expired(scheduler->input, cib_conn,
	options.cib_options, options.rsc_id,
	options.host_uname,
	options.promoted_role_only);

	} else if (options.host_uname) {
	dest = pe_find_node(scheduler->nodes, options.host_uname);
	if (dest == NULL) {
	rc = pcmk_rc_node_unknown;
	if (!out->is_quiet(out)) {
	g_list_free(before);
	}
	return rc;
	}
	rc = cli_resource_clear(options.rsc_id, dest->details->uname, NULL,
	cib_conn, options.cib_options, TRUE, options.force);

	} else {
	rc = cli_resource_clear(options.rsc_id, NULL, scheduler->nodes,
	cib_conn, options.cib_options, TRUE, options.force);
	}

	if (!out->is_quiet(out)) {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("Could not get modified CIB: %s\n"), pcmk_rc_str(rc));
	g_list_free(before);
	free_xml(*cib_xml_copy);
	*cib_xml_copy = NULL;
	return rc;
	}

	scheduler->input = *cib_xml_copy;
	cluster_status(scheduler);

	after = build_constraint_list(scheduler->input);
	remaining = pcmk__subtract_lists(before, after, (GCompareFunc) strcmp);

	for (ele = remaining; ele != NULL; ele = ele->next) {
	out->info(out, "Removing constraint: %s", (char *) ele->data);
	}

	g_list_free(before);
	g_list_free(after);
	g_list_free(remaining);
	}

	return rc;
	}

	static int
	initialize_scheduler_data(xmlNodePtr *cib_xml_copy)
	{
	int rc = pcmk_rc_ok;

	if (options.xml_file != NULL) {
	*cib_xml_copy = filename2xml(options.xml_file);
	if (*cib_xml_copy == NULL) {
	rc = pcmk_rc_cib_corrupt;
	}
	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	}

	if (rc == pcmk_rc_ok) {
	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = ENOMEM;
	} else {
	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts
	\|pcmk_sched_no_compat);
	scheduler->priv = out;
	rc = update_scheduler_input(scheduler, cib_xml_copy);
	}
	}

	if (rc != pcmk_rc_ok) {
	free_xml(*cib_xml_copy);
	*cib_xml_copy = NULL;
	return rc;
	}

	cluster_status(scheduler);
	return pcmk_rc_ok;
	}

	static int
	refresh(pcmk__output_t *out)
	{
	int rc = pcmk_rc_ok;
	const char *router_node = options.host_uname;
	int attr_options = pcmk__node_attr_none;

	if (options.host_uname) {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, options.host_uname);

	if (pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("No cluster connection to Pacemaker Remote node %s detected"),
	options.host_uname);
	return rc;
	}
	router_node = node->details->uname;
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	options.host_uname? options.host_uname : "all nodes");
	rc = pcmk_rc_ok;
	return rc;
	}

	crm_debug("Re-checking the state of all resources on %s", options.host_uname?options.host_uname:"all nodes");

	rc = pcmk__attrd_api_clear_failures(NULL, options.host_uname, NULL,
	NULL, NULL, NULL, attr_options);

	if (pcmk_controld_api_reprobe(controld_api, options.host_uname,
	router_node) == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}

	return rc;
	}

	static void
	refresh_resource(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	int rc = pcmk_rc_ok;

	if (options.force == FALSE) {
	rsc = uber_parent(rsc);
	}

	crm_debug("Re-checking the state of %s (%s requested) on %s",
	rsc->id, options.rsc_id, (options.host_uname? options.host_uname: "all nodes"));
	rc = cli_resource_delete(controld_api, options.host_uname, rsc, NULL, 0,
	FALSE, scheduler, options.force);

	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	// Show any reasons why resource might stay stopped
	cli_resource_check(out, rsc, node);
	}

	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	}

	static int
	set_property(void)
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (pcmk__str_empty(options.rsc_type)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("Must specify -t with resource type"));
	rc = ENXIO;
	return rc;

	} else if (pcmk__str_empty(options.prop_value)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("Must supply -v with new value"));
	rc = ENXIO;
	return rc;
	}

	CRM_LOG_ASSERT(options.prop_name != NULL);

	msg_data = create_xml_node(NULL, options.rsc_type);
	crm_xml_add(msg_data, PCMK_XA_ID, options.rsc_id);
	crm_xml_add(msg_data, options.prop_name, options.prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data,
	options.cib_options);
	rc = pcmk_legacy2rc(rc);
	free_xml(msg_data);

	return rc;
	}

	static int
	show_metadata(pcmk__output_t out, const char agent_spec)
	{
	int rc = pcmk_rc_ok;
	char *standard = NULL;
	char *provider = NULL;
	char *type = NULL;
	char *metadata = NULL;
	lrmd_t *lrmd_conn = NULL;

	rc = lrmd__new(&lrmd_conn, NULL, NULL, 0);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("Could not create executor connection"));
	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	rc = crm_parse_agent_spec(agent_spec, &standard, &provider, &type);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
	provider, type,
	&metadata, 0);
	rc = pcmk_legacy2rc(rc);

	if (metadata) {
	out->output_xml(out, "metadata", metadata);
	free(metadata);
	} else {
	/* We were given a validly formatted spec, but it doesn't necessarily
	* match up with anything that exists. Use ENXIO as the return code
	* here because that maps to an exit code of CRM_EX_NOSUCH, which
	* probably is the most common reason to get here.
	*/
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("Metadata query for %s failed: %s"),
	agent_spec, pcmk_rc_str(rc));
	}
	} else {
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("'%s' is not a valid agent specification"), agent_spec);
	}

	lrmd_api_delete(lrmd_conn);
	return rc;
	}

	static void
	validate_cmdline_config(void)
	{
	// Cannot use both --resource and command-line resource configuration
	if (options.rsc_id != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("--resource cannot be used with --class, --agent, and --provider"));

	// Not all commands support command-line resource configuration
	} else if (options.rsc_cmd != cmd_execute_agent) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("--class, --agent, and --provider can only be used with "
	"--validate and --force-*"));

	// Not all of --class, --agent, and --provider need to be given. Not all
	// classes support the concept of a provider. Check that what we were given
	// is valid.
	} else if (pcmk__str_eq(options.v_class, "stonith", pcmk__str_none)) {
	if (options.v_provider != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("stonith does not support providers"));

	} else if (stonith_agent_exists(options.v_agent, 0) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("%s is not a known stonith agent"), options.v_agent ? options.v_agent : "");
	}

	} else if (resources_agent_exists(options.v_class, options.v_provider, options.v_agent) == FALSE) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	_("%s:%s:%s is not a known resource"),
	options.v_class ? options.v_class : "",
	options.v_provider ? options.v_provider : "",
	options.v_agent ? options.v_agent : "");
	}

	if (error != NULL) {
	return;
	}

	if (options.cmdline_params == NULL) {
	options.cmdline_params = pcmk__strkey_table(free, free);
	}
	options.require_resource = FALSE;
	options.require_scheduler = FALSE;
	options.require_cib = FALSE;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },
	{ "resource", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.rsc_id,
	"Resource ID",
	"ID" },
	{ G_OPTION_REMAINING, 0, G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING_ARRAY, &options.remainder,
	NULL,
	NULL },

	{ NULL }
	};

	const char *description = "Examples:\n\n"
	"List the available OCF agents:\n\n"
	"\t# crm_resource --list-agents ocf\n\n"
	"List the available OCF agents from the linux-ha project:\n\n"
	"\t# crm_resource --list-agents ocf:heartbeat\n\n"
	"Move 'myResource' to a specific node:\n\n"
	"\t# crm_resource --resource myResource --move --node altNode\n\n"
	"Allow (but not force) 'myResource' to move back to its original "
	"location:\n\n"
	"\t# crm_resource --resource myResource --clear\n\n"
	"Stop 'myResource' (and anything that depends on it):\n\n"
	"\t# crm_resource --resource myResource --set-parameter "
	PCMK_META_TARGET_ROLE "--meta --parameter-value Stopped\n\n"
	"Tell the cluster not to manage 'myResource' (the cluster will not "
	"attempt to start or stop the\n"
	"resource under any circumstances; useful when performing maintenance "
	"tasks on a resource):\n\n"
	"\t# crm_resource --resource myResource --set-parameter "
	PCMK_META_IS_MANAGED "--meta --parameter-value false\n\n"
	"Erase the operation history of 'myResource' on 'aNode' (the cluster "
	"will 'forget' the existing\n"
	"resource state, including any errors, and attempt to recover the"
	"resource; useful when a resource\n"
	"had failed permanently and has been repaired by an administrator):\n\n"
	"\t# crm_resource --resource myResource --cleanup --node aNode\n\n";

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	g_option_context_set_description(context, description);

	/* Add the -Q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show query help", query_entries);
	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command help", command_entries);
	pcmk__add_arg_group(context, "locations", "Locations:",
	"Show location help", location_entries);
	pcmk__add_arg_group(context, "advanced", "Advanced:",
	"Show advanced option help", advanced_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	xmlNode *cib_xml_copy = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_node_t *node = NULL;
	int rc = pcmk_rc_ok;

	GOptionGroup *output_group = NULL;
	gchar **processed_args = NULL;
	GOptionContext *context = NULL;

	/*
	* Parse command line arguments
	*/

	args = pcmk__new_common_args(SUMMARY);
	processed_args = pcmk__cmdline_preproc(argv, "GHINSTdginpstuvx");
	context = build_arg_context(args, &output_group);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_resource", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("Error creating output format %s: %s"),
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	pe__register_messages(out);
	crm_resource_register_messages(out);
	lrmd__register_messages(out);
	pcmk__register_lib_messages(out);

	out->quiet = args->quiet;

	crm_log_args(argc, argv);

	/*
	* Validate option combinations
	*/

	// If the user didn't explicitly specify a command, list resources
	if (options.rsc_cmd == cmd_none) {
	options.rsc_cmd = cmd_list_resources;
	options.require_resource = FALSE;
	}

	// --expired without --clear/-U doesn't make sense
	if (options.clear_expired && (options.rsc_cmd != cmd_clear)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, _("--expired requires --clear or -U"));
	goto done;
	}

	if ((options.remainder != NULL) && (options.override_params != NULL)) {
	// Commands that use positional arguments will create override_params
	for (gchar *s = options.remainder; s; s++) {
	char name = calloc(1, strlen(s));
	char value = calloc(1, strlen(s));
	int rc = sscanf(*s, "%[^=]=%s", name, value);

	if (rc == 2) {
	g_hash_table_replace(options.override_params, name, value);

	} else {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Error parsing '%s' as a name=value pair"),
	argv[optind]);
	free(value);
	free(name);
	goto done;
	}
	}

	} else if (options.remainder != NULL) {
	gchar **strv = NULL;
	gchar *msg = NULL;
	int i = 1;
	int len = 0;

	for (gchar *s = options.remainder; s; s++) {
	len++;
	}

	CRM_ASSERT(len > 0);

	/* Add 1 for the strv[0] string below, and add another 1 for the NULL
	* at the end of the array so g_strjoinv knows when to stop.
	*/
	strv = calloc(len+2, sizeof(char *));
	strv[0] = strdup("non-option ARGV-elements:\n");

	for (gchar *s = options.remainder; s; s++) {
	strv[i] = crm_strdup_printf("[%d of %d] %s\n", i, len, *s);
	i++;
	}

	strv[i] = NULL;

	exit_code = CRM_EX_USAGE;
	msg = g_strjoinv("", strv);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
	g_free(msg);

	/* Don't try to free the last element, which is just NULL. */
	for(i = 0; i < len+1; i++) {
	free(strv[i]);
	}
	free(strv);

	goto done;
	}

	if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	/* Kind of a hack to display XML lists using a real tag instead of <list>. This just
	* saves from having to write custom messages to build the lists around all these things
	*/
	switch (options.rsc_cmd) {
	case cmd_execute_agent:
	case cmd_list_resources:
	case cmd_query_xml:
	case cmd_query_raw_xml:
	case cmd_list_active_ops:
	case cmd_list_all_ops:
	case cmd_colocations:
	pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
	break;

	default:
	pcmk__force_args(context, &error, "%s --xml-substitute", g_get_prgname());
	break;
	}
	} else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
	if ((options.rsc_cmd == cmd_colocations) \|\|
	options.rsc_cmd == cmd_list_resources) {
	pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
	}
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (options.cmdline_config) {
	/* A resource configuration was given on the command line. Sanity-check
	* the values and set error if they don't make sense.
	*/
	validate_cmdline_config();
	if (error != NULL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	} else if (options.cmdline_params != NULL) {
	// @COMPAT @TODO error out here when we can break backward compatibility
	g_hash_table_destroy(options.cmdline_params);
	options.cmdline_params = NULL;
	}

	if (options.require_resource && (options.rsc_id == NULL)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Must supply a resource id with -r"));
	goto done;
	}
	if (options.require_node && (options.host_uname == NULL)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Must supply a node name with -N"));
	goto done;
	}

	/*
	* Set up necessary connections
	*/

	if (options.find_flags && options.rsc_id) {
	options.require_scheduler = TRUE;
	}

	// Establish a connection to the CIB if needed
	if (options.require_cib) {
	cib_conn = cib_new();
	if ((cib_conn == NULL) \|\| (cib_conn->cmds == NULL)) {
	exit_code = CRM_EX_DISCONNECT;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Could not create CIB connection"));
	goto done;
	}
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Could not connect to the CIB: %s"), pcmk_rc_str(rc));
	goto done;
	}
	}

	// Populate scheduler data from XML file if specified or CIB query otherwise
	if (options.require_scheduler) {
	rc = initialize_scheduler_data(&cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	goto done;
	}
	}

	// If command requires that resource exist if specified, find it
	if (options.find_flags && options.rsc_id) {
	rsc = pe_find_resource_with_flags(scheduler->resources, options.rsc_id,
	options.find_flags);
	if (rsc == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Resource '%s' not found"), options.rsc_id);
	goto done;
	}

	/* The --ban, --clear, --move, and --restart commands do not work with
	* instances of clone resourcs.
	*/
	if (strchr(options.rsc_id, ':') != NULL && pe_rsc_is_clone(rsc->parent) &&
	(options.rsc_cmd == cmd_ban \|\| options.rsc_cmd == cmd_clear \|\|
	options.rsc_cmd == cmd_move \|\| options.rsc_cmd == cmd_restart)) {
	exit_code = CRM_EX_INVALID_PARAM;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Cannot operate on clone resource instance '%s'"), options.rsc_id);
	goto done;
	}
	}

	// If user supplied a node name, check whether it exists
	if ((options.host_uname != NULL) && (scheduler != NULL)) {
	node = pe_find_node(scheduler->nodes, options.host_uname);

	if (node == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Node '%s' not found"), options.host_uname);
	goto done;
	}
	}

	// Establish a connection to the controller if needed
	if (options.require_crmd) {
	rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Error connecting to the controller: %s"), pcmk_rc_str(rc));
	goto done;
	}
	pcmk_register_ipc_callback(controld_api, controller_event_callback,
	NULL);
	rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Error connecting to %s: %s"),
	pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
	goto done;
	}
	}

	/*
	* Handle requested command
	*/

	switch (options.rsc_cmd) {
	case cmd_list_resources: {
	GList *all = NULL;
	all = g_list_prepend(all, (gpointer) "*");
	rc = out->message(out, "resource-list", scheduler,
	pcmk_show_inactive_rscs \| pcmk_show_rsc_only \| pcmk_show_pending,
	true, all, all, false);
	g_list_free(all);

	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}
	break;
	}

	case cmd_list_instances:
	rc = out->message(out, "resource-names-list", scheduler->resources);

	if (rc != pcmk_rc_ok) {
	rc = ENXIO;
	}

	break;

	case cmd_list_alternatives:
	rc = pcmk__list_alternatives(out, options.agent_spec);
	break;

	case cmd_list_agents:
	rc = pcmk__list_agents(out, options.agent_spec);
	break;

	case cmd_list_standards:
	rc = pcmk__list_standards(out);
	break;

	case cmd_list_providers:
	rc = pcmk__list_providers(out, options.agent_spec);
	break;

	case cmd_metadata:
	rc = show_metadata(out, options.agent_spec);
	break;

	case cmd_restart:
	/* We don't pass scheduler because rsc needs to stay valid for the
	* entire lifetime of cli_resource_restart(), but it will reset and
	* update the scheduler data multiple times, so it needs to use its
	* own copy.
	*/
	rc = cli_resource_restart(out, rsc, node, options.move_lifetime,
	options.timeout_ms, cib_conn,
	options.cib_options, options.promoted_role_only,
	options.force);
	break;

	case cmd_wait:
	rc = wait_till_stable(out, options.timeout_ms, cib_conn);
	break;

	case cmd_execute_agent:
	if (options.cmdline_config) {
	exit_code = cli_resource_execute_from_params(out, NULL,
	options.v_class, options.v_provider, options.v_agent,
	options.operation, options.cmdline_params,
	options.override_params, options.timeout_ms,
	args->verbosity, options.force, options.check_level);
	} else {
	exit_code = cli_resource_execute(rsc, options.rsc_id,
	options.operation, options.override_params,
	options.timeout_ms, cib_conn, scheduler,
	args->verbosity, options.force, options.check_level);
	}
	goto done;

	case cmd_digests:
	node = pe_find_node(scheduler->nodes, options.host_uname);
	if (node == NULL) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = pcmk__resource_digests(out, rsc, node,
	options.override_params);
	}
	break;

	case cmd_colocations:
	rc = out->message(out, "locations-and-colocations", rsc,
	options.recursive, (bool) options.force);
	break;

	case cmd_cts:
	rc = pcmk_rc_ok;
	g_list_foreach(scheduler->resources, (GFunc) cli_resource_print_cts,
	out);
	cli_resource_print_cts_constraints(scheduler);
	break;

	case cmd_fail:
	rc = cli_resource_fail(controld_api, options.host_uname,
	options.rsc_id, scheduler);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	break;

	case cmd_list_active_ops:
	rc = cli_resource_print_operations(options.rsc_id,
	options.host_uname, TRUE,
	scheduler);
	break;

	case cmd_list_all_ops:
	rc = cli_resource_print_operations(options.rsc_id,
	options.host_uname, FALSE,
	scheduler);
	break;

	case cmd_locate: {
	GList *nodes = cli_resource_search(rsc, options.rsc_id, scheduler);
	rc = out->message(out, "resource-search-list", nodes, options.rsc_id);
	g_list_free_full(nodes, free);
	break;
	}

	case cmd_query_xml:
	rc = cli_resource_print(rsc, scheduler, true);
	break;

	case cmd_query_raw_xml:
	rc = cli_resource_print(rsc, scheduler, false);
	break;

	case cmd_why:
	if ((options.host_uname != NULL) && (node == NULL)) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = out->message(out, "resource-reasons-list",
	scheduler->resources, rsc, node);
	}
	break;

	case cmd_clear:
	rc = clear_constraints(out, &cib_xml_copy);
	break;

	case cmd_move:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime);
	} else {
	rc = cli_resource_move(rsc, options.rsc_id, options.host_uname,
	options.move_lifetime, cib_conn,
	options.cib_options, scheduler,
	options.promoted_role_only,
	options.force);
	}

	if (rc == EINVAL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	break;

	case cmd_ban:
	if (options.host_uname == NULL) {
	rc = ban_or_move(out, rsc, options.move_lifetime);
	} else if (node == NULL) {
	rc = pcmk_rc_node_unknown;
	} else {
	rc = cli_resource_ban(out, options.rsc_id, node->details->uname,
	options.move_lifetime, cib_conn,
	options.cib_options,
	options.promoted_role_only,
	PCMK__ROLE_PROMOTED);
	}

	if (rc == EINVAL) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	break;

	case cmd_get_property:
	rc = out->message(out, "property-list", rsc, options.prop_name);
	if (rc == pcmk_rc_no_output) {
	rc = ENXIO;
	}

	break;

	case cmd_set_property:
	rc = set_property();
	break;

	case cmd_get_param: {
	unsigned int count = 0;
	GHashTable *params = NULL;
	pcmk_node_t *current = rsc->fns->active_node(rsc, &count, NULL);
	bool free_params = true;
	const char* value = NULL;

	if (count > 1) {
	out->err(out, "%s is active on more than one node,"
	" returning the default value for %s", rsc->id,
	pcmk__s(options.prop_name, "unspecified property"));
	current = NULL;
	}

	crm_debug("Looking up %s in %s", options.prop_name, rsc->id);

	if (pcmk__str_eq(options.attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_none)) {
	params = pe_rsc_params(rsc, current, scheduler);
	free_params = false;

	value = g_hash_table_lookup(params, options.prop_name);

	} else if (pcmk__str_eq(options.attr_set_type, XML_TAG_META_SETS, pcmk__str_none)) {
	params = pcmk__strkey_table(free, free);
	get_meta_attributes(params, rsc, current, scheduler);

	value = g_hash_table_lookup(params, options.prop_name);

	} else if (pcmk__str_eq(options.attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {

	value = crm_element_value(rsc->xml, options.prop_name);
	free_params = false;

	} else {
	params = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_UTILIZATION, NULL, params,
	NULL, FALSE, scheduler);

	value = g_hash_table_lookup(params, options.prop_name);
	}

	rc = out->message(out, "attribute-list", rsc, options.prop_name, value);
	if (free_params) {
	g_hash_table_destroy(params);
	}

	break;
	}

	case cmd_set_param:
	if (pcmk__str_empty(options.prop_value)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("You need to supply a value with the -v option"));
	goto done;
	}

	/* coverity[var_deref_model] False positive */
	rc = cli_resource_update_attribute(rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name,
	options.prop_value,
	options.recursive, cib_conn,
	options.cib_options,
	options.force);
	break;

	case cmd_delete_param:
	/* coverity[var_deref_model] False positive */
	rc = cli_resource_delete_attribute(rsc, options.rsc_id,
	options.prop_set,
	options.attr_set_type,
	options.prop_id,
	options.prop_name, cib_conn,
	options.cib_options,
	options.force);
	break;

	case cmd_cleanup:
	if (rsc == NULL) {
	rc = cli_cleanup_all(controld_api, options.host_uname,
	options.operation, options.interval_spec,
	scheduler);
	if (rc == pcmk_rc_ok) {
	start_mainloop(controld_api);
	}
	} else {
	cleanup(out, rsc, node);
	}
	break;

	case cmd_refresh:
	if (rsc == NULL) {
	rc = refresh(out);
	} else {
	refresh_resource(out, rsc, node);
	}
	break;

	case cmd_delete:
	/* rsc_id was already checked for NULL much earlier when validating
	* command line arguments.
	*/
	if (options.rsc_type == NULL) {
	// @COMPAT @TODO change this to exit_code = CRM_EX_USAGE
	rc = ENXIO;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("You need to specify a resource type with -t"));
	} else {
	rc = pcmk__resource_delete(cib_conn, options.cib_options,
	options.rsc_id, options.rsc_type);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	_("Could not delete resource %s: %s"),
	options.rsc_id, pcmk_rc_str(rc));
	}
	}

	break;

	default:
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Unimplemented command: %d"), (int) options.rsc_cmd);
	goto done;
	}

	/* Convert rc into an exit code. */
	if (rc != pcmk_rc_ok && rc != pcmk_rc_no_output) {
	exit_code = pcmk_rc2exitc(rc);
	}

	/*
	* Clean up and exit
	*/

	done:
	/* When we get here, exit_code has been set one of two ways - either at one of
	* the spots where there's a "goto done" (which itself could have happened either
	* directly or by calling pcmk_rc2exitc), or just up above after any of the break
	* statements.
	*
	* Thus, we can use just exit_code here to decide what to do.
	*/
	if (exit_code != CRM_EX_OK && exit_code != CRM_EX_USAGE) {
	if (error != NULL) {
	char *msg = crm_strdup_printf("%s\nError performing operation: %s",
	error->message, crm_exit_str(exit_code));
	g_clear_error(&error);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "%s", msg);
	free(msg);
	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	_("Error performing operation: %s"), crm_exit_str(exit_code));
	}
	}

	g_free(options.host_uname);
	g_free(options.interval_spec);
	g_free(options.move_lifetime);
	g_free(options.operation);
	g_free(options.prop_id);
	free(options.prop_name);
	g_free(options.prop_set);
	g_free(options.prop_value);
	g_free(options.rsc_id);
	g_free(options.rsc_type);
	free(options.agent_spec);
	free(options.v_agent);
	free(options.v_class);
	free(options.v_provider);
	g_free(options.xml_file);
	g_strfreev(options.remainder);

	if (options.override_params != NULL) {
	g_hash_table_destroy(options.override_params);
	}

	/* options.cmdline_params does not need to be destroyed here. See the
	* comments in cli_resource_execute_from_params.
	*/

	g_strfreev(processed_args);
	g_option_context_free(context);

	return bye(exit_code);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index e4ec45d47e..64505a755f 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2255 +1,2255 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/common/ipc_attrd_internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	static GList *
	build_node_info_list(const pcmk_resource_t *rsc)
	{
	GList *retval = NULL;

	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;

	for (const GList *iter2 = child->running_on;
	iter2 != NULL; iter2 = iter2->next) {

	const pcmk_node_t node = (const pcmk_node_t ) iter2->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));

	ni->node_name = node->details->uname;
	ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
	child->fns->state(child, TRUE) == pcmk_role_promoted;

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pcmk_resource_t rsc, const char requested_name,
	pcmk_scheduler_t *scheduler)
	{
	GList *retval = NULL;
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	if (pe_rsc_is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	retval = build_node_info_list(parent);

	} else if (rsc->running_on != NULL) {
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	GString *xpath = NULL;
	const char *xpath_base = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath,
	xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);

	if (attr_set_type != NULL) {
	pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
	if (set_name != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
	NULL);
	}
	}

	g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
	if (attr_id != NULL) {
	pcmk__g_strcat(xpath, "@" PCMK_XA_ID "=\"", attr_id, "\"", NULL);
	}

	if (attr_name != NULL) {
	if (attr_id != NULL) {
	g_string_append(xpath, " and ");
	}
	pcmk__g_strcat(xpath, "@" XML_NVPAIR_ATTR_NAME "=\"", attr_name, "\"",
	NULL);
	}
	g_string_append_c(xpath, ']');

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	xmlNode *child = NULL;

	rc = ENOTUNIQ;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}

	done:
	g_string_free(xpath, TRUE);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t *out,
	GList /* <pcmk_resource_t> / **result,
	pcmk_resource_t rsc, const char rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result,
	(pcmk_resource_t *) gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if (rsc->variant == pcmk_rsc_variant_clone) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pcmk_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pcmk_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if ((rsc->parent != NULL)
	&& (rsc->parent->variant == pcmk_rsc_variant_clone)) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);

	} else if ((rsc->parent == NULL) && (rsc->children != NULL)
	&& (rsc->variant == pcmk_rsc_variant_clone)) {
	pcmk_resource_t *child = rsc->children->data;

	if (child->variant == pcmk_rsc_variant_primitive) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
	attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;

	char *found_attr_id = NULL;

	GList/<pcmk_resource_t>/ resources = NULL;
	const char *top_id = pe__const_top_resource(rsc, false)->id;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
	attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (!force) {
	rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &found_attr_id);
	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	out->err(out,
	"WARNING: There is already a meta attribute "
	"for '%s' called '%s' (id=%s)",
	top_id, attr_name, found_attr_id);
	out->err(out,
	" Delete '%s' first or use the force option "
	"to override", found_attr_id);
	}
	free(found_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	crm_xml_add(rsc->xml, attr_name, attr_value);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set attribute: name=%s value=%s",
	attr_name, attr_value);
	}
	return rc;

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"update", force);
	}

	/* If the user specified attr_set or attr_id, the intent is to modify a
	* single resource, which will be the last item in the list.
	*/
	if ((attr_set != NULL) \|\| (attr_id != NULL)) {
	GList *last = g_list_last(resources);

	resources = g_list_remove_link(resources, last);
	g_list_free(resources);
	resources = last;
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	char *local_attr_set = NULL;
	const char *rsc_attr_id = attr_id;
	const char *rsc_attr_set = attr_set;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	found_attr_id = NULL;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);

	switch (rc) {
	case pcmk_rc_ok:
	crm_debug("Found a match for name=%s: id=%s",
	attr_name, found_attr_id);
	rsc_attr_id = found_attr_id;
	break;

	case ENXIO:
	if (rsc_attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	rsc_attr_set = local_attr_set;
	}
	if (rsc_attr_id == NULL) {
	found_attr_id = crm_strdup_printf("%s-%s",
	rsc_attr_set, attr_name);
	rsc_attr_id = found_attr_id;
	}

	xml_top = create_xml_node(NULL, (const char *) rsc->xml->name);
	crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
	break;

	default:
	free(lookup_id);
	free(found_attr_id);
	g_list_free(resources);
	return rc;
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
	attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
	lookup_id, found_attr_id,
	((rsc_attr_set == NULL)? "" : " set="),
	pcmk__s(rsc_attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""), attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(found_attr_id);
	free(local_attr_set);

	if (recursive
	&& pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	GList *lpc = NULL;
	static bool need_init = true;

	if (need_init) {
	need_init = false;
	pcmk__unpack_constraints(rsc->cluster);
	pe__clear_resource_flags_on_all(rsc->cluster,
	pcmk_rsc_detect_loop);
	}

	/* We want to set the attribute only on resources explicitly
	* colocated with this one, so we use rsc->rsc_cons_lhs directly
	* rather than the with_this_colocations() method.
	*/
	pe__set_resource_flags(rsc, pcmk_rsc_detect_loop);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (!pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
	&& (cons->score > 0)) {
	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	cli_resource_update_attribute(cons->dependent,
	cons->dependent->id, NULL,
	attr_set_type, NULL,
	attr_name, attr_value,
	recursive, cib, cib_options,
	force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;
	GList/<pcmk_resource_t>/ resources = NULL;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID,
	pe__const_top_resource(rsc, false)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"delete", force);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	xml_remove_prop(rsc->xml, attr_name);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted attribute: %s", attr_name);
	}
	return rc;

	} else {
	resources = g_list_append(resources, rsc);
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *found_attr_id = NULL;
	const char *rsc_attr_id = attr_id;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);
	switch (rc) {
	case pcmk_rc_ok:
	break;

	case ENXIO:
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	default:
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (rsc_attr_id == NULL) {
	rsc_attr_id = found_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
	cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
	lookup_id, found_attr_id,
	((attr_set == NULL)? "" : " set="),
	pcmk__s(attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""));
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(found_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pcmk_rsc_variant_primitive) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pcmk_node_t *node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	return strdup(name);
	}
	return clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	guint interval_ms = 0U;

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(scheduler->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pcmk_resource_t *fail_rsc = NULL;

	fail_rsc = pe_find_resource_with_flags(scheduler->resources,
	failed_id,
	pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename);
	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	free(interval_ms_s);

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(const pcmk_resource_t rsc, const char operation,
	const char interval_spec, const pcmk_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	const pcmk_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pcmk_scheduler_t *scheduler, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	pcmk_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {

	for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, scheduler,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(scheduler->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pcmk_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->details->uname, rsc,
	operation, interval_spec, just_failures,
	scheduler, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, scheduler);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_rc_str(rc));
	return rc;
	}
	} else {
	for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_rc_str(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	static void
	check_role(resource_checks_t *checks)
	{
	const char *role_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_TARGET_ROLE);

	if (role_s == NULL) {
	return;
	}
	switch (text2role(role_s)) {
	case pcmk_role_stopped:
	checks->flags \|= rsc_remain_stopped;
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
	pcmk_rsc_promotable)) {
	checks->flags \|= rsc_unpromotable;
	}
	break;

	default:
	break;
	}
	}

	static void
	check_managed(resource_checks_t *checks)
	{
	const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_IS_MANAGED);

	if ((managed_s != NULL) && !crm_is_true(managed_s)) {
	checks->flags \|= rsc_unmanaged;
	}
	}

	static void
	check_locked(resource_checks_t *checks)
	{
	if (checks->rsc->lock_node != NULL) {
	checks->flags \|= rsc_locked;
	checks->lock_node = checks->rsc->lock_node->details->uname;
	}
	}

	static bool
	node_is_unhealthy(pcmk_node_t *node)
	{
	switch (pe__health_strategy(node->details->data_set)) {
	case pcmk__health_strategy_none:
	break;

	case pcmk__health_strategy_no_red:
	if (pe__node_health(node) < 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_only_green:
	if (pe__node_health(node) <= 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_progressive:
	case pcmk__health_strategy_custom:
	/* @TODO These are finite scores, possibly with rules, and possibly
	* combining with other scores, so attributing these as a cause is
	* nontrivial.
	*/
	break;
	}
	return false;
	}

	static void
	check_node_health(resource_checks_t checks, pcmk_node_t node)
	{
	if (node == NULL) {
	GHashTableIter iter;
	bool allowed = false;
	bool all_nodes_unhealthy = true;

	g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	allowed = true;
	if (!node_is_unhealthy(node)) {
	all_nodes_unhealthy = false;
	break;
	}
	}
	if (allowed && all_nodes_unhealthy) {
	checks->flags \|= rsc_node_health;
	}

	} else if (node_is_unhealthy(node)) {
	checks->flags \|= rsc_node_health;
	}
	}

	int
	cli_resource_check(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	resource_checks_t checks = { .rsc = rsc };

	check_role(&checks);
	check_managed(&checks);
	check_locked(&checks);
	check_node_health(&checks, node);

	return out->message(out, "resource-check-list", &checks);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
	}

	static GHashTable *
	generate_resource_params(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, scheduler);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, node, scheduler);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pcmk_resource_t rsc, const char host)
	{
	bool found = true;
	GList *hIter = NULL;
	GList *hosts = NULL;

	if (rsc == NULL) {
	return false;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pcmk_node_t node = (pcmk_node_t ) hIter->data;

	if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if (host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = false;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = false;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pcmk_rsc_variant_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as scheduler input
	*
	* This also updates the scheduler timestamp to the current time.
	*
	* \param[in,out] scheduler Scheduler data to update
	* \param[in,out] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_scheduler_input(pcmk_scheduler_t scheduler, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	scheduler->input = *xml;
	scheduler->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update scheduler XML input based on a CIB query
	*
	* \param[in] scheduler Scheduler data to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->input and scheduler->now.
	*/
	static int
	update_scheduler_input_to_cib(pcmk__output_t out, pcmk_scheduler_t scheduler,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
	return rc;
	}
	rc = update_scheduler_input(scheduler, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pcmk_scheduler_t scheduler, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = scheduler->priv;

	pe_reset_working_set(scheduler);
	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = write_xml_file(scheduler->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	pcmk__simulate_transition(scheduler, shadow_cib, NULL);
	out->quiet = prev_quiet;

	rc = update_dataset(shadow_cib, scheduler, false);

	} else {
	cluster_status(scheduler);
	}

	done:
	// Do not free scheduler->input here, we need rsc->xml to be valid later on
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Find the maximum stop timeout of a resource and its children (if any)
	*
	* \param[in,out] rsc Resource to get timeout for
	*
	* \return Maximum stop timeout for \p rsc (in milliseconds)
	*/
	static int
	max_rsc_stop_timeout(pcmk_resource_t *rsc)
	{
	long long result_ll;
	int max_delay = 0;
	xmlNode *config = NULL;
	GHashTable *meta = NULL;

	if (rsc == NULL) {
	return 0;
	}

	// If resource is collective, use maximum of its children's stop timeouts
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	int delay = max_rsc_stop_timeout(child);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Maximum stop timeout for %s is now %s "
	"due to %s", rsc->id,
	pcmk__readable_interval(delay), child->id);
	max_delay = delay;
	}
	}
	return max_delay;
	}

	// Get resource's stop action configuration from CIB
	config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);

	/* Get configured timeout for stop action (fully evaluated for rules,
	* defaults, etc.).
	*
	* @TODO This currently ignores node (which might matter for rules)
	*/
	meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
	if ((pcmk__scan_ll(g_hash_table_lookup(meta, XML_ATTR_TIMEOUT),
	&result_ll, -1LL) == pcmk_rc_ok)
	&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
	max_delay = (int) result_ll;
	}
	g_hash_table_destroy(meta);

	return max_delay;
	}

	/*!
	* \internal
	* \brief Find a reasonable waiting time for stopping any one resource in a list
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] resources List of names of resources that will be stopped
	*
	* \return Rough estimate of a reasonable time to wait (in seconds) to stop any
	* one resource in \p resources
	* \note This estimate is very rough, simply the maximum stop timeout of all
	* given resources and their children, plus a small fudge factor. It does
	* not account for children that must be stopped in sequence, action
	* throttling, or any demotions needed. It checks the stop timeout, even
	* if the resources in question are actually being started.
	*/
	static int
	wait_time_estimate(pcmk_scheduler_t scheduler, const GList resources)
	{
	int max_delay = 0;

	// Find maximum stop timeout in milliseconds
	for (const GList *item = resources; item != NULL; item = item->next) {
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
	(const char *) item->data);
	int delay = max_rsc_stop_timeout(rsc);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Wait time is now %s due to %s",
	pcmk__readable_interval(delay), rsc->id);
	max_delay = delay;
	}
	}

	return (max_delay / 1000) + 5;
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in,out] out Output object
	* \param[in,out] rsc The resource to restart
	* \param[in] node Node to restart resource on (NULL for all)
	* \param[in] move_lifetime If not NULL, how long constraint should
	* remain in effect (as ISO 8601 string)
	* \param[in] timeout_ms Consider failed if actions do not complete
	* in this time (specified in milliseconds,
	* but a two-second granularity is actually
	* used; if 0, it will be calculated based on
	* the resource timeout)
	* \param[in,out] cib Connection to the CIB manager
	* \param[in] cib_options Group of enum cib_call_options flags to
	* use with CIB calls
	* \param[in] promoted_role_only If true, limit to promoted instances
	* \param[in] force If true, apply only to requested instance
	* if part of a collective resource
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pcmk_resource_t rsc,
	const pcmk_node_t node, const char move_lifetime,
	int timeout_ms, cib_t *cib, int cib_options,
	gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = false;
	char *rsc_id = NULL;
	char *lookup_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pcmk_scheduler_t *scheduler = NULL;
	pcmk_resource_t *parent = uber_parent(rsc);

	bool running = false;
	const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
	const char *host = node ? node->details->uname : NULL;

	/* If the implicit resource or primitive resource of a bundle is given, operate on the
	* bundle itself instead.
	*/
	if (pe_rsc_is_bundled(rsc)) {
	rsc = parent->parent;
	}

	running = resource_is_running_on(rsc, host);

	if (pe_rsc_is_clone(parent) && !running) {
	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	rsc = parent->fns->find_rsc(parent, lookup_id, node,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node);
	free(lookup_id);
	running = resource_is_running_on(rsc, host);
	}

	if (!running) {
	if (host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	out->err(out, "Unmanaged resources cannot be restarted.");
	return EAGAIN;
	}

	rsc_id = strdup(rsc->id);

	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	if (host) {
	if (pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) {
	stop_via_ban = true;
	} else if (pe_rsc_is_clone(parent)) {
	stop_via_ban = true;
	free(lookup_id);
	lookup_id = strdup(parent->id);
	}
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	scheduler->priv = out;
	rc = update_dataset(cib, scheduler, false);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, scheduler->resources);
	current_active = get_active_resources(host, scheduler->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
	cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else {
	/* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
	* Remember any existing PCMK_META_TARGET_ROLE so we can restore it
	* later (though it only makes any difference if it's Unpromoted).
	*/

	find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
	NULL, PCMK_META_TARGET_ROLE, &orig_target_role);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	PCMK_ACTION_STOPPED, FALSE, cib,
	cib_options, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	current_active = NULL;
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	restart_target_active = NULL;
	}
	goto done;
	}

	rc = update_dataset(cib, scheduler, true);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, scheduler->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, scheduler, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);

	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out,
	"Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	goto done;
	}

	if (target_active != NULL) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;

	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, scheduler, false);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(NULL, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	PCMK_META_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	done:
	if (list_delta != NULL) {
	g_list_free(list_delta);
	}
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	if (target_active != NULL && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	free(lookup_id);
	pe_free_working_set(scheduler);
	return rc;
	}

	static inline bool
	action_is_pending(const pcmk_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pcmk_action_runnable)
	\|\| pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return true if any actions in the list are pending, otherwise false
	*/
	static bool
	actions_are_pending(const GList *actions)
	{
	for (const GList *action = actions; action != NULL; action = action->next) {
	const pcmk_action_t a = (const pcmk_action_t ) action->data;

	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	return true;
	}
	}
	return false;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pcmk_action_t a = (pcmk_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s",
	a->id, a->uuid, pe__node_name(a->node));
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in,out] out Output object
	* \param[in] timeout_ms Consider failed if actions do not complete in
	* this time (specified in milliseconds, but
	* one-second granularity is actually used; if 0, a
	* default will be used)
	* \param[in,out] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlXPathObjectPtr search;
	int rc = pcmk_rc_ok;
	bool pending_unknown_state_resources;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	return ENOMEM;
	}

	do {
	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
	} else {
	print_pending_actions(out, scheduler->actions);
	pe_free_working_set(scheduler);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(scheduler);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(scheduler);
	return rc;
	}
	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_DC_VERSION);

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = true;
	}
	}

	search = xpath_search(scheduler->input, "/cib/status/node_state/lrm/lrm_resources/lrm_resource/"
	XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_RC "='193']");
	pending_unknown_state_resources = (numXpathResults(search) > 0);
	freeXpathObject(search);
	} while (actions_are_pending(scheduler->actions) \|\| pending_unknown_state_resources);

	pe_free_working_set(scheduler);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = PCMK_ACTION_VALIDATE_ALL;

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = PCMK_ACTION_MONITOR;

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in,out] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	g_hash_table_insert(params, strdup(PCMK_XA_CRM_FEATURE_SET),
	strdup(CRM_FEATURE_SET));

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in,out] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	g_hash_table_replace(params, strdup(name), strdup(value));
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force,
	int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, timeout_ms, params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of the %s standard is "
	"unsupported", pcmk__s(class, "unspecified"));
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pcmk_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t cib, pcmk_scheduler_t scheduler,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = scheduler->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if(pe_rsc_is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out,
	"Try setting "
	PCMK_META_TARGET_ROLE "=" PCMK__ROLE_STOPPED
	" first or specifying the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if (rsc->variant == pcmk_rsc_variant_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (pe_rsc_is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	scheduler);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action),
	scheduler);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(const pcmk_resource_t rsc, const char rsc_id,
	const char host_name, const char move_lifetime, cib_t *cib,
	int cib_options, pcmk_scheduler_t *scheduler,
	gboolean promoted_role_only, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pcmk_node_t *current = NULL;
	pcmk_node_t *dest = pe_find_node(scheduler->nodes, host_name);
	bool cur_is_dest = false;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {

	const pcmk_resource_t *p = pe__const_top_resource(rsc, false);

	if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring --promoted option: %s is not promotable",
	rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	unsigned int promoted_count = 0;
	pcmk_node_t *promoted_node = NULL;

	for (const GList *iter = rsc->children; iter; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == pcmk_role_promoted) {
	rsc = child;
	promoted_node = pe__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (current && (current->details == dest->details)) {
	cur_is_dest = true;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active",
	pe__node_name(dest));
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
	force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);

	crm_trace("%s%s now prefers %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	pe__node_name(dest), force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if (force && !cur_is_dest) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, pe__node_name(dest));
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}
	diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
	index aab4110e60..15f183361f 100644
	--- a/tools/crm_simulate.c
	+++ b/tools/crm_simulate.c
	@@ -1,590 +1,590 @@
	/*
	* Copyright 2009-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <time.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#define SUMMARY "crm_simulate - simulate a Pacemaker cluster's response to events"

	struct {
	char *dot_file;
	char *graph_file;
	gchar *input_file;
	pcmk_injections_t *injections;
	unsigned int flags;
	gchar *output_file;
	long long repeat;
	gboolean store;
	gchar *test_dir;
	char *use_date;
	char *xml_file;
	} options = {
	.flags = pcmk_sim_show_pending \| pcmk_sim_sanitized,
	.repeat = 1
	};

	uint32_t section_opts = 0;
	char *temp_shadow = NULL;
	crm_exit_t exit_code = CRM_EX_OK;

	#define INDENT " "

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static gboolean
	all_actions_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_all_actions;
	return TRUE;
	}

	static gboolean
	attrs_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	section_opts \|= pcmk_section_attributes;
	return TRUE;
	}

	static gboolean
	failcounts_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	section_opts \|= pcmk_section_failcounts \| pcmk_section_failures;
	return TRUE;
	}

	static gboolean
	in_place_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.store = TRUE;
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	return TRUE;
	}

	static gboolean
	live_check_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (options.xml_file) {
	free(options.xml_file);
	}

	options.xml_file = NULL;
	options.flags &= ~pcmk_sim_sanitized;
	return TRUE;
	}

	static gboolean
	node_down_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->node_down = g_list_append(options.injections->node_down, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	node_fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->node_fail = g_list_append(options.injections->node_fail, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	node_up_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__simulate_node_config = true;
	options.injections->node_up = g_list_append(options.injections->node_up, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	op_fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	options.injections->op_fail = g_list_append(options.injections->op_fail, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	op_inject_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->op_inject = g_list_append(options.injections->op_inject, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	pending_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_show_pending;
	return TRUE;
	}

	static gboolean
	process_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	return TRUE;
	}

	static gboolean
	quorum_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.injections->quorum, optarg);
	return TRUE;
	}

	static gboolean
	save_dotfile_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	pcmk__str_update(&options.dot_file, optarg);
	return TRUE;
	}

	static gboolean
	save_graph_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	pcmk__str_update(&options.graph_file, optarg);
	return TRUE;
	}

	static gboolean
	show_scores_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_show_scores;
	return TRUE;
	}

	static gboolean
	simulate_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	return TRUE;
	}

	static gboolean
	ticket_activate_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_activate = g_list_append(options.injections->ticket_activate, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_grant_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_grant = g_list_append(options.injections->ticket_grant, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_revoke_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_revoke = g_list_append(options.injections->ticket_revoke, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_standby_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_standby = g_list_append(options.injections->ticket_standby, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	utilization_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_show_utilization;
	return TRUE;
	}

	static gboolean
	watchdog_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.injections->watchdog, optarg);
	return TRUE;
	}

	static gboolean
	xml_file_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.xml_file, optarg);
	options.flags \|= pcmk_sim_sanitized;
	return TRUE;
	}

	static gboolean
	xml_pipe_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.xml_file, "-");
	options.flags \|= pcmk_sim_sanitized;
	return TRUE;
	}

	static GOptionEntry operation_entries[] = {
	{ "run", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, process_cb,
	"Process the supplied input and show what actions the cluster will take in response",
	NULL },
	{ "simulate", 'S', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, simulate_cb,
	"Like --run, but also simulate taking those actions and show the resulting new status",
	NULL },
	{ "in-place", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, in_place_cb,
	"Like --simulate, but also store the results back to the input file",
	NULL },
	{ "show-attrs", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attrs_cb,
	"Show node attributes",
	NULL },
	{ "show-failcounts", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, failcounts_cb,
	"Show resource fail counts",
	NULL },
	{ "show-scores", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_scores_cb,
	"Show allocation scores",
	NULL },
	{ "show-utilization", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, utilization_cb,
	"Show utilization information",
	NULL },
	{ "profile", 'P', 0, G_OPTION_ARG_FILENAME, &options.test_dir,
	"Process all the XML files in the named directory to create profiling data",
	"DIR" },
	{ "repeat", 'N', 0, G_OPTION_ARG_INT, &options.repeat,
	"With --profile, repeat each test N times and print timings",
	"N" },
	/* Deprecated */
	{ "pending", 'j', G_OPTION_FLAG_NO_ARG\|G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, pending_cb,
	"Display pending state if 'record-pending' is enabled",
	NULL },

	{ NULL }
	};

	static GOptionEntry synthetic_entries[] = {
	{ "node-up", 'u', 0, G_OPTION_ARG_CALLBACK, node_up_cb,
	"Simulate bringing a node online",
	"NODE" },
	{ "node-down", 'd', 0, G_OPTION_ARG_CALLBACK, node_down_cb,
	"Simulate taking a node offline",
	"NODE" },
	{ "node-fail", 'f', 0, G_OPTION_ARG_CALLBACK, node_fail_cb,
	"Simulate a node failing",
	"NODE" },
	{ "op-inject", 'i', 0, G_OPTION_ARG_CALLBACK, op_inject_cb,
	"Generate a failure for the cluster to react to in the simulation.\n"
	INDENT "See `Operation Specification` help for more information.",
	"OPSPEC" },
	{ "op-fail", 'F', 0, G_OPTION_ARG_CALLBACK, op_fail_cb,
	"If the specified task occurs during the simulation, have it fail with return code ${rc}.\n"
	INDENT "The transition will normally stop at the failed action.\n"
	INDENT "Save the result with --save-output and re-run with --xml-file.\n"
	INDENT "See `Operation Specification` help for more information.",
	"OPSPEC" },
	{ "set-datetime", 't', 0, G_OPTION_ARG_STRING, &options.use_date,
	"Set date/time (ISO 8601 format, see https://en.wikipedia.org/wiki/ISO_8601)",
	"DATETIME" },
	{ "quorum", 'q', 0, G_OPTION_ARG_CALLBACK, quorum_cb,
	"Set to '1' (or 'true') to indicate cluster has quorum",
	"QUORUM" },
	{ "watchdog", 'w', 0, G_OPTION_ARG_CALLBACK, watchdog_cb,
	"Set to '1' (or 'true') to indicate cluster has an active watchdog device",
	"DEVICE" },
	{ "ticket-grant", 'g', 0, G_OPTION_ARG_CALLBACK, ticket_grant_cb,
	"Simulate granting a ticket",
	"TICKET" },
	{ "ticket-revoke", 'r', 0, G_OPTION_ARG_CALLBACK, ticket_revoke_cb,
	"Simulate revoking a ticket",
	"TICKET" },
	{ "ticket-standby", 'b', 0, G_OPTION_ARG_CALLBACK, ticket_standby_cb,
	"Simulate making a ticket standby",
	"TICKET" },
	{ "ticket-activate", 'e', 0, G_OPTION_ARG_CALLBACK, ticket_activate_cb,
	"Simulate activating a ticket",
	"TICKET" },

	{ NULL }
	};

	static GOptionEntry artifact_entries[] = {
	{ "save-input", 'I', 0, G_OPTION_ARG_FILENAME, &options.input_file,
	"Save the input configuration to the named file",
	"FILE" },
	{ "save-output", 'O', 0, G_OPTION_ARG_FILENAME, &options.output_file,
	"Save the output configuration to the named file",
	"FILE" },
	{ "save-graph", 'G', 0, G_OPTION_ARG_CALLBACK, save_graph_cb,
	"Save the transition graph (XML format) to the named file",
	"FILE" },
	{ "save-dotfile", 'D', 0, G_OPTION_ARG_CALLBACK, save_dotfile_cb,
	"Save the transition graph (DOT format) to the named file",
	"FILE" },
	{ "all-actions", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, all_actions_cb,
	"Display all possible actions in DOT graph (even if not part of transition)",
	NULL },

	{ NULL }
	};

	static GOptionEntry source_entries[] = {
	{ "live-check", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, live_check_cb,
	"Connect to CIB manager and use the current CIB contents as input",
	NULL },
	{ "xml-file", 'x', 0, G_OPTION_ARG_CALLBACK, xml_file_cb,
	"Retrieve XML from the named file",
	"FILE" },
	{ "xml-pipe", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, xml_pipe_cb,
	"Retrieve XML from stdin",
	NULL },

	{ NULL }
	};

	static int
	setup_input(pcmk__output_t out, const char input, const char *output,
	GError **error)
	{
	int rc = pcmk_rc_ok;
	xmlNode *cib_object = NULL;
	char *local_output = NULL;

	if (input == NULL) {
	/* Use live CIB */
	rc = cib__signon_query(out, NULL, &cib_object);
	if (rc != pcmk_rc_ok) {
	// cib__signon_query() outputs any relevant error
	return rc;
	}

	} else if (pcmk__str_eq(input, "-", pcmk__str_casei)) {
	cib_object = filename2xml(NULL);

	} else {
	cib_object = filename2xml(input);
	}

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return pcmk_rc_transform_failed;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return pcmk_rc_schema_validation;
	}

	if (output == NULL) {
	char *pid = pcmk__getpid_s();

	local_output = get_shadow_file(pid);
	temp_shadow = strdup(local_output);
	output = local_output;
	free(pid);
	}

	rc = write_xml_file(cib_object, output, FALSE);
	free_xml(cib_object);
	cib_object = NULL;

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT,
	"Could not create '%s': %s", output, pcmk_rc_str(rc));
	return rc;
	} else {
	setenv("CIB_file", output, 1);
	free(local_output);
	return pcmk_rc_ok;
	}
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Display only essential output",
	NULL },

	{ NULL }
	};

	const char *description = "Operation Specification:\n\n"
	"The OPSPEC in any command line option is of the form\n"
	"${resource}_${task}_${interval_in_ms}@${node}=${rc}\n"
	"(memcached_monitor_20000@bart.example.com=7, for example).\n"
	"${rc} is an OCF return code. For more information on these\n"
	"return codes, refer to https://clusterlabs.org/pacemaker/doc/2.1/Pacemaker_Administration/html/agents.html#ocf-return-codes\n\n"
	"Examples:\n\n"
	"Pretend a recurring monitor action found memcached stopped on node\n"
	"fred.example.com and, during recovery, that the memcached stop\n"
	"action failed:\n\n"
	"\tcrm_simulate -LS --op-inject memcached:0_monitor_20000@bart.example.com=7 "
	"--op-fail memcached:0_stop_0@fred.example.com=1 --save-output /tmp/memcached-test.xml\n\n"
	"Now see what the reaction to the stop failed would be:\n\n"
	"\tcrm_simulate -S --xml-file /tmp/memcached-test.xml\n\n";

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	pcmk__add_main_args(context, extra_prog_entries);
	g_option_context_set_description(context, description);

	pcmk__add_arg_group(context, "operations", "Operations:",
	"Show operations options", operation_entries);
	pcmk__add_arg_group(context, "synthetic", "Synthetic Cluster Events:",
	"Show synthetic cluster event options", synthetic_entries);
	pcmk__add_arg_group(context, "artifact", "Artifact Options:",
	"Show artifact options", artifact_entries);
	pcmk__add_arg_group(context, "source", "Data Source:",
	"Show data source options", source_entries);

	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	pcmk_scheduler_t *scheduler = NULL;
	pcmk__output_t *out = NULL;

	GError *error = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINOP");
	GOptionContext *context = build_arg_context(args, &output_group);

	options.injections = calloc(1, sizeof(pcmk_injections_t));
	if (options.injections == NULL) {
	rc = ENOMEM;
	goto done;
	}

	/* This must come before g_option_context_parse_strv. */
	options.xml_file = strdup("-");

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_simulate", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	fprintf(stderr, "Error creating output format %s: %s\n",
	args->output_ty, pcmk_rc_str(rc));
	exit_code = CRM_EX_ERROR;
	goto done;
	}

	if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches) &&
	!pcmk_is_set(options.flags, pcmk_sim_show_scores) &&
	!pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
	pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
	} else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	out->quiet = args->quiet;

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (args->verbosity > 0) {
	options.flags \|= pcmk_sim_verbose;

	#ifdef PCMK__COMPAT_2_0
	/* Redirect stderr to stdout so we can grep the output */
	close(STDERR_FILENO);
	dup2(STDOUT_FILENO, STDERR_FILENO);
	#endif
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = ENOMEM;
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not allocate scheduler data");
	goto done;
	}

	if (pcmk_is_set(options.flags, pcmk_sim_show_scores)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_output_scores);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(scheduler, pcmk_sched_show_utilization);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}
	- pe__set_working_set_flags(scheduler, pcmk_sched_no_compat);
	+ pcmk__set_scheduler_flags(scheduler, pcmk_sched_no_compat);

	if (options.test_dir != NULL) {
	scheduler->priv = out;
	pcmk__profile_dir(options.test_dir, options.repeat, scheduler,
	options.use_date);
	rc = pcmk_rc_ok;
	goto done;
	}

	rc = setup_input(out, options.xml_file,
	options.store? options.xml_file : options.output_file,
	&error);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk__simulate(scheduler, out, options.injections, options.flags,
	section_opts, options.use_date, options.input_file,
	options.graph_file, options.dot_file);

	done:
	pcmk__output_and_clear_error(&error, NULL);

	/* There sure is a lot to free in options. */
	free(options.dot_file);
	free(options.graph_file);
	g_free(options.input_file);
	g_free(options.output_file);
	g_free(options.test_dir);
	free(options.use_date);
	free(options.xml_file);

	pcmk_free_injections(options.injections);
	pcmk__free_arg_context(context);
	g_strfreev(processed_args);

	if (scheduler != NULL) {
	pe_free_working_set(scheduler);
	}

	fflush(stderr);

	if (temp_shadow) {
	unlink(temp_shadow);
	free(temp_shadow);
	}

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	}

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}

	pcmk__unregister_formats();
	crm_exit(exit_code);
	}
	diff --git a/tools/crm_ticket.c b/tools/crm_ticket.c
	index 4adc5f676d..61255a66c0 100644
	--- a/tools/crm_ticket.c
	+++ b/tools/crm_ticket.c
	@@ -1,1011 +1,1011 @@
	/*
	* Copyright 2012-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/cmdline_internal.h>

	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>

	#include <pacemaker-internal.h>

	GError *error = NULL;

	#define SUMMARY "Perform tasks related to cluster tickets\n\n" \
	"Allows ticket attributes to be queried, modified and deleted."

	struct {
	gchar *attr_default;
	gchar *attr_id;
	char *attr_name;
	char *attr_value;
	gboolean force;
	char *get_attr_name;
	gboolean quiet;
	gchar *set_name;
	char ticket_cmd;
	gchar *ticket_id;
	gchar *xml_file;
	} options = {
	.ticket_cmd = 'S'
	};

	GList *attr_delete;
	GHashTable *attr_set;
	bool modified = false;
	int cib_options = cib_sync_call;

	#define INDENT " "

	static gboolean
	attr_value_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.attr_value, optarg);

	if (!options.attr_name \|\| !options.attr_value) {
	return TRUE;
	}

	g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
	pcmk__str_update(&options.attr_name, NULL);
	pcmk__str_update(&options.attr_value, NULL);

	modified = true;

	return TRUE;
	}

	static gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (pcmk__str_any_of(option_name, "--info", "-l", NULL)) {
	options.ticket_cmd = 'l';
	} else if (pcmk__str_any_of(option_name, "--details", "-L", NULL)) {
	options.ticket_cmd = 'L';
	} else if (pcmk__str_any_of(option_name, "--raw", "-w", NULL)) {
	options.ticket_cmd = 'w';
	} else if (pcmk__str_any_of(option_name, "--query-xml", "-q", NULL)) {
	options.ticket_cmd = 'q';
	} else if (pcmk__str_any_of(option_name, "--constraints", "-c", NULL)) {
	options.ticket_cmd = 'c';
	} else if (pcmk__str_any_of(option_name, "--cleanup", "-C", NULL)) {
	options.ticket_cmd = 'C';
	}

	return TRUE;
	}

	static gboolean
	delete_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	attr_delete = g_list_append(attr_delete, strdup(optarg));
	modified = true;
	return TRUE;
	}

	static gboolean
	get_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.get_attr_name, optarg);
	options.ticket_cmd = 'G';
	return TRUE;
	}

	static gboolean
	grant_standby_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (pcmk__str_any_of(option_name, "--grant", "-g", NULL)) {
	g_hash_table_insert(attr_set, strdup("granted"), strdup("true"));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--revoke", "-r", NULL)) {
	g_hash_table_insert(attr_set, strdup("granted"), strdup("false"));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--standby", "-s", NULL)) {
	g_hash_table_insert(attr_set, strdup("standby"), strdup("true"));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--activate", "-a", NULL)) {
	g_hash_table_insert(attr_set, strdup("standby"), strdup("false"));
	modified = true;
	}

	return TRUE;
	}

	static gboolean
	set_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.attr_name, optarg);

	if (!options.attr_name \|\| !options.attr_value) {
	return TRUE;
	}

	g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
	pcmk__str_update(&options.attr_name, NULL);
	pcmk__str_update(&options.attr_value, NULL);

	modified = true;

	return TRUE;
	}

	static GOptionEntry query_entries[] = {
	{ "info", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the information of ticket(s)",
	NULL },

	{ "details", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the details of ticket(s)",
	NULL },

	{ "raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the IDs of ticket(s)",
	NULL },

	{ "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Query the XML of ticket(s)",
	NULL },

	{ "constraints", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the rsc_ticket constraints that apply to ticket(s)",
	NULL },

	{ NULL }
	};

	static GOptionEntry command_entries[] = {
	{ "grant", 'g', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Grant a ticket to this cluster site",
	NULL },

	{ "revoke", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Revoke a ticket from this cluster site",
	NULL },

	{ "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Tell this cluster site this ticket is standby",
	NULL },

	{ "activate", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Tell this cluster site this ticket is active",
	NULL },

	{ NULL }
	};

	static GOptionEntry advanced_entries[] = {
	{ "get-attr", 'G', 0, G_OPTION_ARG_CALLBACK, get_attr_cb,
	"Display the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "set-attr", 'S', 0, G_OPTION_ARG_CALLBACK, set_attr_cb,
	"Set the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "delete-attr", 'D', 0, G_OPTION_ARG_CALLBACK, delete_attr_cb,
	"Delete the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Delete all state of a ticket at this cluster site",
	NULL },

	{ NULL}
	};

	static GOptionEntry addl_entries[] = {
	{ "attr-value", 'v', 0, G_OPTION_ARG_CALLBACK, attr_value_cb,
	"Attribute value to use with -S",
	"VALUE" },

	{ "default", 'd', 0, G_OPTION_ARG_STRING, &options.attr_default,
	"(Advanced) Default attribute value to display if none is found\n"
	INDENT "(for use with -G)",
	"VALUE" },

	{ "force", 'f', 0, G_OPTION_ARG_NONE, &options.force,
	"(Advanced) Force the action to be performed",
	NULL },

	{ "ticket", 't', 0, G_OPTION_ARG_STRING, &options.ticket_id,
	"Ticket ID",
	"ID" },

	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.xml_file,
	NULL,
	NULL },

	{ NULL }
	};

	static GOptionEntry deprecated_entries[] = {
	{ "set-name", 'n', 0, G_OPTION_ARG_STRING, &options.set_name,
	"(Advanced) ID of the instance_attributes object to change",
	"ID" },

	{ "nvpair", 'i', 0, G_OPTION_ARG_STRING, &options.attr_id,
	"(Advanced) ID of the nvpair object to change/delete",
	"ID" },

	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &options.quiet,
	"Print only the value on stdout",
	NULL },

	{ NULL }
	};

	static pcmk_ticket_t *
	find_ticket(gchar ticket_id, pcmk_scheduler_t scheduler)
	{
	return g_hash_table_lookup(scheduler->tickets, ticket_id);
	}

	static void
	print_date(time_t time)
	{
	int lpc = 0;
	char date_str[26];

	asctime_r(localtime(&time), date_str);
	for (; lpc < 26; lpc++) {
	if (date_str[lpc] == '\n') {
	date_str[lpc] = 0;
	}
	}
	fprintf(stdout, "'%s'", date_str);
	}

	static void
	print_ticket(pcmk_ticket_t *ticket, bool raw, bool details)
	{
	if (raw) {
	fprintf(stdout, "%s\n", ticket->id);
	return;
	}

	fprintf(stdout, "%s\t%s %s",
	ticket->id, ticket->granted ? "granted" : "revoked",
	ticket->standby ? "[standby]" : " ");

	if (details && g_hash_table_size(ticket->state) > 0) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;
	int lpc = 0;

	fprintf(stdout, " (");

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void )&name, (void )&value)) {
	if (lpc > 0) {
	fprintf(stdout, ", ");
	}
	fprintf(stdout, "%s=", name);
	if (pcmk__str_any_of(name, "last-granted", "expires", NULL)) {
	long long time_ll;

	pcmk__scan_ll(value, &time_ll, 0);
	print_date((time_t) time_ll);
	} else {
	fprintf(stdout, "%s", value);
	}
	lpc++;
	}

	fprintf(stdout, ")\n");

	} else {
	if (ticket->last_granted > -1) {
	fprintf(stdout, " last-granted=");
	print_date(ticket->last_granted);
	}
	fprintf(stdout, "\n");
	}

	return;
	}

	static void
	print_ticket_list(pcmk_scheduler_t *scheduler, bool raw, bool details)
	{
	GHashTableIter iter;
	pcmk_ticket_t *ticket = NULL;

	g_hash_table_iter_init(&iter, scheduler->tickets);

	while (g_hash_table_iter_next(&iter, NULL, (void **)&ticket)) {
	print_ticket(ticket, raw, details);
	}
	}

	static int
	find_ticket_state(cib_t * the_cib, gchar ticket_id, xmlNode * ticket_state_xml)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = NULL;

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	xpath = g_string_sized_new(1024);
	g_string_append(xpath,
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
	"/" XML_CIB_TAG_TICKETS);

	if (ticket_id != NULL) {
	pcmk__g_strcat(xpath,
	"/" XML_CIB_TAG_TICKET_STATE
	"[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
	}

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	if (ticket_id) {
	fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
	}
	*ticket_state_xml = xml_search;
	} else {
	*ticket_state_xml = xml_search;
	}
	return rc;
	}

	static int
	find_ticket_constraints(cib_t * the_cib, gchar ticket_id, xmlNode * ticket_cons_xml)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = NULL;
	const char *xpath_base = NULL;

	CRM_ASSERT(ticket_cons_xml != NULL);
	*ticket_cons_xml = NULL;

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_CONSTRAINTS);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_CONSTRAINTS " CIB element not known (bug?)");
	return -ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath, xpath_base, "/" XML_CONS_TAG_RSC_TICKET, NULL);

	if (ticket_id != NULL) {
	pcmk__g_strcat(xpath,
	"[@" XML_TICKET_ATTR_TICKET "=\"", ticket_id, "\"]",
	NULL);
	}

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	*ticket_cons_xml = xml_search;

	return rc;
	}

	static int
	dump_ticket_xml(cib_t * the_cib, gchar *ticket_id)
	{
	int rc = pcmk_rc_ok;
	xmlNode *state_xml = NULL;

	rc = find_ticket_state(the_cib, ticket_id, &state_xml);

	if (state_xml == NULL) {
	return rc;
	}

	fprintf(stdout, "State XML:\n");
	if (state_xml) {
	char *state_xml_str = NULL;

	state_xml_str = dump_xml_formatted(state_xml);
	fprintf(stdout, "\n%s", state_xml_str);
	free_xml(state_xml);
	free(state_xml_str);
	}

	return rc;
	}

	static int
	dump_constraints(cib_t * the_cib, gchar *ticket_id)
	{
	int rc = pcmk_rc_ok;
	xmlNode *cons_xml = NULL;
	char *cons_xml_str = NULL;

	rc = find_ticket_constraints(the_cib, ticket_id, &cons_xml);

	if (cons_xml == NULL) {
	return rc;
	}

	cons_xml_str = dump_xml_formatted(cons_xml);
	fprintf(stdout, "Constraints XML:\n\n%s", cons_xml_str);
	free_xml(cons_xml);
	free(cons_xml_str);

	return rc;
	}

	static int
	get_ticket_state_attr(gchar ticket_id, const char attr_name, const char **attr_value,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_ticket_t *ticket = NULL;

	CRM_ASSERT(attr_value != NULL);
	*attr_value = NULL;

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	return ENXIO;
	}

	*attr_value = g_hash_table_lookup(ticket->state, attr_name);
	if (*attr_value == NULL) {
	return ENXIO;
	}

	return pcmk_rc_ok;
	}

	static void
	ticket_warning(gchar ticket_id, const char action)
	{
	GString *warning = g_string_sized_new(1024);
	const char *word = NULL;

	CRM_ASSERT(action != NULL);

	if (strcmp(action, "grant") == 0) {
	pcmk__g_strcat(warning,
	"This command cannot help you verify whether '",
	ticket_id,
	"' has been already granted elsewhere.\n", NULL);
	word = "to";

	} else {
	pcmk__g_strcat(warning,
	"Revoking '", ticket_id, "' can trigger the specified "
	"'loss-policy'(s) relating to '", ticket_id, "'.\n\n"
	"You can check that with:\n"
	"crm_ticket --ticket ", ticket_id, " --constraints\n\n"
	"Otherwise before revoking '", ticket_id, "', "
	"you may want to make '", ticket_id, "' "
	"standby with:\n"
	"crm_ticket --ticket ", ticket_id, " --standby\n\n",
	NULL);
	word = "from";
	}

	pcmk__g_strcat(warning,
	"If you really want to ", action, " '", ticket_id, "' ",
	word, " this site now, and you know what you are doing,\n"
	"please specify --force.", NULL);

	fprintf(stdout, "%s\n", (const char *) warning->str);

	g_string_free(warning, TRUE);
	}

	static bool
	allow_modification(gchar *ticket_id)
	{
	const char *value = NULL;
	GList *list_iter = NULL;

	if (options.force) {
	return true;
	}

	if (g_hash_table_lookup_extended(attr_set, "granted", NULL, (gpointer *) & value)) {
	if (crm_is_true(value)) {
	ticket_warning(ticket_id, "grant");
	return false;

	} else {
	ticket_warning(ticket_id, "revoke");
	return false;
	}
	}

	for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
	const char key = (const char )list_iter->data;

	if (pcmk__str_eq(key, "granted", pcmk__str_casei)) {
	ticket_warning(ticket_id, "revoke");
	return false;
	}
	}

	return true;
	}

	static int
	modify_ticket_state(gchar ticket_id, cib_t cib, pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;
	bool found = false;

	GList *list_iter = NULL;
	GHashTableIter hash_iter;

	char *key = NULL;
	char *value = NULL;

	pcmk_ticket_t *ticket = NULL;

	rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
	if (rc == pcmk_rc_ok) {
	crm_debug("Found a match state for ticket: id=%s", ticket_id);
	xml_top = ticket_state_xml;
	found = true;

	} else if (rc != ENXIO) {
	return rc;

	} else if (g_hash_table_size(attr_set) == 0){
	return pcmk_rc_ok;

	} else {
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);
	}

	for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
	const char key = (const char )list_iter->data;
	xml_remove_prop(ticket_state_xml, key);
	}

	ticket = find_ticket(ticket_id, scheduler);

	g_hash_table_iter_init(&hash_iter, attr_set);
	while (g_hash_table_iter_next(&hash_iter, (gpointer ) & key, (gpointer ) & value)) {
	crm_xml_add(ticket_state_xml, key, value);

	if (pcmk__str_eq(key, "granted", pcmk__str_casei)
	&& (ticket == NULL \|\| ticket->granted == FALSE)
	&& crm_is_true(value)) {

	char *now = pcmk__ttoa(time(NULL));

	crm_xml_add(ticket_state_xml, "last-granted", now);
	free(now);
	}
	}

	if (found && (attr_delete != NULL)) {
	crm_log_xml_debug(xml_top, "Replace");
	rc = cib->cmds->replace(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
	rc = pcmk_legacy2rc(rc);

	} else {
	crm_log_xml_debug(xml_top, "Update");
	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);
	}

	free_xml(xml_top);
	return rc;
	}

	static int
	delete_ticket_state(gchar ticket_id, cib_t cib)
	{
	xmlNode *ticket_state_xml = NULL;

	int rc = pcmk_rc_ok;

	rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);

	if (rc == ENXIO) {
	return pcmk_rc_ok;

	} else if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(ticket_state_xml, "Delete");

	rc = cib->cmds->remove(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	fprintf(stdout, "Cleaned up %s\n", ticket_id);
	}

	free_xml(ticket_state_xml);
	return rc;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t *args) {
	GOptionContext *context = NULL;

	const char *description = "Examples:\n\n"
	"Display the info of tickets:\n\n"
	"\tcrm_ticket --info\n\n"
	"Display the detailed info of tickets:\n\n"
	"\tcrm_ticket --details\n\n"
	"Display the XML of 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --query-xml\n\n"
	"Display the rsc_ticket constraints that apply to 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --constraints\n\n"
	"Grant 'ticketA' to this cluster site:\n\n"
	"\tcrm_ticket --ticket ticketA --grant\n\n"
	"Revoke 'ticketA' from this cluster site:\n\n"
	"\tcrm_ticket --ticket ticketA --revoke\n\n"
	"Make 'ticketA' standby (the cluster site will treat a granted\n"
	"'ticketA' as 'standby', and the dependent resources will be\n"
	"stopped or demoted gracefully without triggering loss-policies):\n\n"
	"\tcrm_ticket --ticket ticketA --standby\n\n"
	"Activate 'ticketA' from being standby:\n\n"
	"\tcrm_ticket --ticket ticketA --activate\n\n"
	"Get the value of the 'granted' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --get-attr granted\n\n"
	"Set the value of the 'standby' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --set-attr standby --attr-value true\n\n"
	"Delete the 'granted' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --delete-attr granted\n\n"
	"Erase the operation history of 'ticketA' at this cluster site,\n"
	"causing the cluster site to 'forget' the existing ticket state:\n\n"
	"\tcrm_ticket --ticket ticketA --cleanup\n\n";

	context = pcmk__build_arg_context(args, NULL, NULL, NULL);
	g_option_context_set_description(context, description);

	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show queries", query_entries);
	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command options", command_entries);
	pcmk__add_arg_group(context, "advanced", "Advanced Options:",
	"Show advanced options", advanced_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
	"Show deprecated options", deprecated_entries);

	return context;
	}

	int
	main(int argc, char **argv)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlNode *cib_xml_copy = NULL;

	cib_t *cib_conn = NULL;
	crm_exit_t exit_code = CRM_EX_OK;
	int rc = pcmk_rc_ok;

	pcmk__common_args_t *args = NULL;
	GOptionContext *context = NULL;
	gchar **processed_args = NULL;

	attr_set = pcmk__strkey_table(free, free);
	attr_delete = NULL;

	args = pcmk__new_common_args(SUMMARY);
	context = build_arg_context(args);
	processed_args = pcmk__cmdline_preproc(argv, "dintvxCDGS");

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_ticket", args->verbosity);

	if (args->version) {
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	/* FIXME: When crm_ticket is converted to use formatted output, this can go. */
	pcmk__cli_help('v');
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}
	- pe__set_working_set_flags(scheduler,
	+ pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);

	cib_conn = cib_new();
	if (cib_conn == NULL) {
	exit_code = CRM_EX_DISCONNECT;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB manager");
	goto done;
	}

	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	if (options.xml_file != NULL) {
	cib_xml_copy = filename2xml(options.xml_file);

	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not get local CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}
	}

	if (!cli_config_update(&cib_xml_copy, NULL, FALSE)) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not update local CIB to latest schema version");
	goto done;
	}

	scheduler->input = cib_xml_copy;
	scheduler->now = crm_time_new(NULL);

	cluster_status(scheduler);

	/* For recording the tickets that are referenced in rsc_ticket constraints
	* but have never been granted yet. */
	pcmk__unpack_constraints(scheduler);

	if (options.ticket_cmd == 'l' \|\| options.ticket_cmd == 'L' \|\| options.ticket_cmd == 'w') {
	bool raw = false;
	bool details = false;

	if (options.ticket_cmd == 'L') {
	details = true;
	} else if (options.ticket_cmd == 'w') {
	raw = true;
	}

	if (options.ticket_id) {
	pcmk_ticket_t *ticket = find_ticket(options.ticket_id, scheduler);

	if (ticket == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"No such ticket '%s'", options.ticket_id);
	goto done;
	}
	print_ticket(ticket, raw, details);

	} else {
	print_ticket_list(scheduler, raw, details);
	}

	} else if (options.ticket_cmd == 'q') {
	rc = dump_ticket_xml(cib_conn, options.ticket_id);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not query ticket XML: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'c') {
	rc = dump_constraints(cib_conn, options.ticket_id);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not show ticket constraints: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'G') {
	const char *value = NULL;

	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	rc = get_ticket_state_attr(options.ticket_id, options.get_attr_name,
	&value, scheduler);
	if (rc == pcmk_rc_ok) {
	fprintf(stdout, "%s\n", value);
	} else if (rc == ENXIO && options.attr_default) {
	fprintf(stdout, "%s\n", options.attr_default);
	rc = pcmk_rc_ok;
	}
	exit_code = pcmk_rc2exitc(rc);

	} else if (options.ticket_cmd == 'C') {
	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (options.force == FALSE) {
	pcmk_ticket_t *ticket = NULL;

	ticket = find_ticket(options.ticket_id, scheduler);
	if (ticket == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"No such ticket '%s'", options.ticket_id);
	goto done;
	}

	if (ticket->granted) {
	ticket_warning(options.ticket_id, "revoke");
	exit_code = CRM_EX_INSUFFICIENT_PRIV;
	goto done;
	}
	}

	rc = delete_ticket_state(options.ticket_id, cib_conn);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not clean up ticket: %s", pcmk_rc_str(rc));
	}

	} else if (modified) {
	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (options.attr_value
	&& (pcmk__str_empty(options.attr_name))) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply attribute name with -S for -v %s", options.attr_value);
	goto done;
	}

	if (options.attr_name
	&& (pcmk__str_empty(options.attr_value))) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply attribute value with -v for -S %s", options.attr_value);
	goto done;
	}

	if (!allow_modification(options.ticket_id)) {
	exit_code = CRM_EX_INSUFFICIENT_PRIV;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Ticket modification not allowed");
	goto done;
	}

	rc = modify_ticket_state(options.ticket_id, cib_conn, scheduler);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not modify ticket: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'S') {
	/* Correct usage was handled in the "if (modified)" block above, so
	* this is just for reporting usage errors
	*/

	if (pcmk__str_empty(options.attr_name)) {
	// We only get here if ticket_cmd was left as default
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Must supply a command");
	goto done;
	}

	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (pcmk__str_empty(options.attr_value)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply value with -v for -S %s", options.attr_name);
	goto done;
	}

	} else {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Unknown command: %c", options.ticket_cmd);
	}

	done:
	if (attr_set) {
	g_hash_table_destroy(attr_set);
	}
	attr_set = NULL;

	if (attr_delete) {
	g_list_free_full(attr_delete, free);
	}
	attr_delete = NULL;

	pe_free_working_set(scheduler);
	scheduler = NULL;

	cib__clean_up_connection(&cib_conn);

	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	g_free(options.attr_default);
	g_free(options.attr_id);
	free(options.attr_name);
	free(options.attr_value);
	free(options.get_attr_name);
	g_free(options.set_name);
	g_free(options.ticket_id);
	g_free(options.xml_file);

	pcmk__output_and_clear_error(&error, NULL);

	crm_exit(exit_code);
	}

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jul 20, 9:16 PM (16 h, 59 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2081517
Default Alt Text: (651 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions