No OneTemporary
Actions

Size

130 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/schedulerd/schedulerd_messages.c b/daemons/schedulerd/schedulerd_messages.c
	index 120af8ac47..c5d887190b 100644
	--- a/daemons/schedulerd/schedulerd_messages.c
	+++ b/daemons/schedulerd/schedulerd_messages.c
	@@ -1,335 +1,335 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include <stdbool.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include "pacemaker-schedulerd.h"

	static GHashTable *schedulerd_handlers = NULL;

	static pe_working_set_t *
	init_working_set(void)
	{
	pe_working_set_t *data_set = pe_new_working_set();

	CRM_ASSERT(data_set != NULL);

	crm_config_error = FALSE;
	crm_config_warning = FALSE;

	was_processing_error = FALSE;
	was_processing_warning = FALSE;

	data_set->priv = logger_out;
	return data_set;
	}

	static xmlNode *
	handle_pecalc_request(pcmk__request_t *request)
	{
	static struct series_s {
	const char *name;
	const char *param;

	/* Maximum number of inputs of this kind to save to disk.
	* If -1, save all; if 0, save none.
	*/
	int wrap;
	} series[] = {
	{ "pe-error", "pe-error-series-max", -1 },
	{ "pe-warn", "pe-warn-series-max", 5000 },
	{ "pe-input", "pe-input-series-max", 4000 },
	};

	xmlNode *msg = request->xml;
	xmlNode *xml_data = get_message_xml(msg, F_CRM_DATA);

	static char *last_digest = NULL;
	static char *filename = NULL;

	unsigned int seq;
	int series_id = 0;
	int series_wrap = 0;
	char *digest = NULL;
	const char *value = NULL;
	time_t execution_date = time(NULL);
	xmlNode *converted = NULL;
	xmlNode *reply = NULL;
	bool is_repoke = false;
	bool process = true;
	pe_working_set_t *data_set = init_working_set();

	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INDETERMINATE);

	digest = calculate_xml_versioned_digest(xml_data, FALSE, FALSE,
	CRM_FEATURE_SET);
	converted = copy_xml(xml_data);
	if (!cli_config_update(&converted, NULL, TRUE)) {
	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
	crm_xml_add_int(data_set->graph, "transition_id", 0);
	crm_xml_add_int(data_set->graph, "cluster-delay", 0);
	process = false;
	free(digest);

	} else if (pcmk__str_eq(digest, last_digest, pcmk__str_casei)) {
	is_repoke = true;
	free(digest);

	} else {
	free(last_digest);
	last_digest = digest;
	}

	if (process) {
	pcmk__schedule_actions(converted,
	pcmk_sched_no_counts
	\|pcmk_sched_no_compat
	- \|pe_flag_show_utilization, data_set);
	+ \|pcmk_sched_show_utilization, data_set);
	}

	// Get appropriate index into series[] array
	if (was_processing_error) {
	series_id = 0;
	} else if (was_processing_warning) {
	series_id = 1;
	} else {
	series_id = 2;
	}

	value = pe_pref(data_set->config_hash, series[series_id].param);
	if ((value == NULL)
	\|\| (pcmk__scan_min_int(value, &series_wrap, -1) != pcmk_rc_ok)) {
	series_wrap = series[series_id].wrap;
	}

	if (pcmk__read_series_sequence(PE_STATE_DIR, series[series_id].name,
	&seq) != pcmk_rc_ok) {
	// @TODO maybe handle errors better ...
	seq = 0;
	}
	crm_trace("Series %s: wrap=%d, seq=%u, pref=%s",
	series[series_id].name, series_wrap, seq, value);

	data_set->input = NULL;
	reply = create_reply(msg, data_set->graph);

	if (reply == NULL) {
	pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
	"Failed building ping reply for client %s",
	pcmk__client_name(request->ipc_client));
	goto done;
	}

	if (series_wrap == 0) { // Don't save any inputs of this kind
	free(filename);
	filename = NULL;

	} else if (!is_repoke) { // Input changed, save to disk
	free(filename);
	filename = pcmk__series_filename(PE_STATE_DIR,
	series[series_id].name, seq, true);
	}

	crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename);
	crm_xml_add_int(reply, PCMK__XA_GRAPH_ERRORS, was_processing_error);
	crm_xml_add_int(reply, PCMK__XA_GRAPH_WARNINGS, was_processing_warning);
	crm_xml_add_int(reply, PCMK__XA_CONFIG_ERRORS, crm_config_error);
	crm_xml_add_int(reply, PCMK__XA_CONFIG_WARNINGS, crm_config_warning);

	pcmk__log_transition_summary(filename);

	if (series_wrap == 0) {
	crm_debug("Not saving input to disk (disabled by configuration)");

	} else if (is_repoke) {
	crm_info("Input has not changed since last time, not saving to disk");

	} else {
	unlink(filename);
	crm_xml_add_ll(xml_data, "execution-date", (long long) execution_date);
	write_xml_file(xml_data, filename, TRUE);
	pcmk__write_series_sequence(PE_STATE_DIR, series[series_id].name,
	++seq, series_wrap);
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);

	done:
	free_xml(converted);
	pe_free_working_set(data_set);

	return reply;
	}

	static xmlNode *
	handle_unknown_request(pcmk__request_t *request)
	{
	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INVALID_PARAM);

	pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
	"Unknown IPC request type '%s' (bug?)",
	pcmk__client_name(request->ipc_client));
	return NULL;
	}

	static xmlNode *
	handle_hello_request(pcmk__request_t *request)
	{
	pcmk__ipc_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags,
	"ack", NULL, CRM_EX_INDETERMINATE);

	crm_trace("Received IPC hello from %s", pcmk__client_name(request->ipc_client));

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	static void
	schedulerd_register_handlers(void)
	{
	pcmk__server_command_t handlers[] = {
	{ CRM_OP_HELLO, handle_hello_request },
	{ CRM_OP_PECALC, handle_pecalc_request },
	{ NULL, handle_unknown_request },
	};

	schedulerd_handlers = pcmk__register_handlers(handlers);
	}

	static int32_t
	pe_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static int32_t
	pe_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	xmlNode *msg = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *sys_to = NULL;

	CRM_CHECK(c != NULL, return 0);

	if (schedulerd_handlers == NULL) {
	schedulerd_register_handlers();
	}

	msg = pcmk__client_data2xml(c, data, &id, &flags);
	if (msg == NULL) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
	return 0;
	}

	sys_to = crm_element_value(msg, F_CRM_SYS_TO);

	if (pcmk__str_eq(crm_element_value(msg, F_CRM_MSG_TYPE),
	XML_ATTR_RESPONSE, pcmk__str_none)) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
	crm_info("Ignoring IPC reply from %s", pcmk__client_name(c));

	} else if (!pcmk__str_eq(sys_to, CRM_SYSTEM_PENGINE, pcmk__str_none)) {
	pcmk__ipc_send_ack(c, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);
	crm_info("Ignoring invalid IPC message: to '%s' not "
	CRM_SYSTEM_PENGINE, pcmk__s(sys_to, ""));

	} else {
	char *log_msg = NULL;
	const char *reason = NULL;
	xmlNode *reply = NULL;

	pcmk__request_t request = {
	.ipc_client = c,
	.ipc_id = id,
	.ipc_flags = flags,
	.peer = NULL,
	.xml = msg,
	.call_options = 0,
	.result = PCMK__UNKNOWN_RESULT,
	};

	request.op = crm_element_value_copy(request.xml, F_CRM_TASK);
	CRM_CHECK(request.op != NULL, return 0);

	reply = pcmk__process_request(&request, schedulerd_handlers);

	if (reply != NULL) {
	pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event);
	free_xml(reply);
	}

	reason = request.result.exit_reason;

	log_msg = crm_strdup_printf("Processed %s request from %s %s: %s%s%s%s",
	request.op, pcmk__request_origin_type(&request),
	pcmk__request_origin(&request),
	pcmk_exec_status_str(request.result.execution_status),
	(reason == NULL)? "" : " (",
	(reason == NULL)? "" : reason,
	(reason == NULL)? "" : ")");

	if (!pcmk__result_ok(&request.result)) {
	crm_warn("%s", log_msg);
	} else {
	crm_debug("%s", log_msg);
	}

	free(log_msg);
	pcmk__reset_request(&request);
	}

	free_xml(msg);
	return 0;
	}

	/* Error code means? */
	static int32_t
	pe_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	pe_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	pe_ipc_closed(c);
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = pe_ipc_accept,
	.connection_created = NULL,
	.msg_process = pe_ipc_dispatch,
	.connection_closed = pe_ipc_closed,
	.connection_destroyed = pe_ipc_destroy
	};
	diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
	index ee103772c1..af9823ac54 100644
	--- a/include/crm/pengine/pe_types.h
	+++ b/include/crm/pengine/pe_types.h
	@@ -1,486 +1,484 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_PE_TYPES__H
	# define PCMK__CRM_PENGINE_PE_TYPES__H


	# include <stdbool.h> // bool
	# include <sys/types.h> // time_t
	# include <libxml/tree.h> // xmlNode
	# include <glib.h> // gboolean, guint, GList, GHashTable
	# include <crm/common/iso8601.h>
	# include <crm/common/scheduler.h>
	# include <crm/pengine/common.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/*!
	* \file
	* \brief Data types for cluster status
	* \ingroup pengine
	*/

	typedef struct pe_node_s pe_node_t;
	typedef struct pe_action_s pe_action_t;
	typedef struct pe_resource_s pe_resource_t;
	typedef struct pe_working_set_s pe_working_set_t;

	typedef struct resource_object_functions_s {
	gboolean (unpack) (pe_resource_t, pe_working_set_t*);
	pe_resource_t (find_rsc) (pe_resource_t parent, const char search,
	const pe_node_t *node, int flags);
	/* parameter result must be free'd */
	char (parameter) (pe_resource_t, pe_node_t, gboolean, const char*,
	pe_working_set_t*);
	//! \deprecated will be removed in a future release
	void (print) (pe_resource_t, const char, long, void);
	gboolean (active) (pe_resource_t, gboolean);
	enum rsc_role_e (state) (const pe_resource_t, gboolean);
	pe_node_t (location) (const pe_resource_t, GList*, int);
	void (free) (pe_resource_t);
	void (count) (pe_resource_t);
	gboolean (is_filtered) (const pe_resource_t, GList *, gboolean);

	/*!
	* \brief Find a node (and optionally count all) where resource is active
	*
	* \param[in] rsc Resource to check
	* \param[out] count_all If not NULL, set this to count of active nodes
	* \param[out] count_clean If not NULL, set this to count of clean nodes
	*
	* \return A node where the resource is active, preferring the source node
	* if the resource is involved in a partial migration or a clean,
	* online node if the resource's "requires" is "quorum" or
	* "nothing", or NULL if the resource is inactive.
	*/
	pe_node_t (active_node)(const pe_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean);

	/*!
	* \brief Get maximum resource instances per node
	*
	* \param[in] rsc Resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int (max_per_node)(const pe_resource_t rsc);
	} resource_object_functions_t;

	typedef struct resource_alloc_functions_s resource_alloc_functions_t;

	-# define pe_flag_show_utilization 0x04000000ULL
	-
	/*!
	* When scheduling, only unpack the CIB (including constraints), calculate
	* as much cluster status as possible, and apply node health.
	*/
	# define pe_flag_check_config 0x08000000ULL

	struct pe_working_set_s {
	xmlNode *input;
	crm_time_t *now;

	/* options extracted from the input */
	char *dc_uuid;
	pe_node_t *dc_node;
	const char *stonith_action;
	const char *placement_strategy;

	unsigned long long flags;

	int stonith_timeout;
	enum pe_quorum_policy no_quorum_policy;

	GHashTable *config_hash;
	GHashTable *tickets;

	// Actions for which there can be only one (e.g. fence nodeX)
	GHashTable *singletons;

	GList *nodes;
	GList *resources;
	GList *placement_constraints;
	GList *ordering_constraints;
	GList *colocation_constraints;
	GList *ticket_constraints;

	GList *actions;
	xmlNode *failed;
	xmlNode *op_defaults;
	xmlNode *rsc_defaults;

	/* stats */
	int num_synapse;
	int max_valid_nodes; //! Deprecated (will be removed in a future release)
	int order_id;
	int action_id;

	/* final output */
	xmlNode *graph;

	GHashTable *template_rsc_sets;
	const char *localhost;
	GHashTable *tags;

	int blocked_resources;
	int disabled_resources;

	GList *param_check; // History entries that need to be checked
	GList *stop_needed; // Containers that need stop actions
	time_t recheck_by; // Hint to controller to re-run scheduler by this time
	int ninstances; // Total number of resource instances
	guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
	int priority_fencing_delay; // Priority fencing delay

	void *priv;
	guint node_pending_timeout; // Node pending timeout
	};

	struct pe_node_shared_s {
	const char *id;
	const char *uname;
	enum node_type type;

	/* @TODO convert these flags into a bitfield */
	gboolean online;
	gboolean standby;
	gboolean standby_onfail;
	gboolean pending;
	gboolean unclean;
	gboolean unseen;
	gboolean shutdown;
	gboolean expected_up;
	gboolean is_dc;
	gboolean maintenance;
	gboolean rsc_discovery_enabled;
	gboolean remote_requires_reset;
	gboolean remote_was_fenced;
	gboolean remote_maintenance; /* what the remote-rsc is thinking */
	gboolean unpacked;

	int num_resources;
	pe_resource_t *remote_rsc;
	GList running_rsc; / pe_resource_t* */
	GList allocated_rsc; / pe_resource_t* */

	GHashTable attrs; / char* => char* */
	GHashTable *utilization;
	GHashTable *digest_cache; //!< cache of calculated resource digests
	int priority; // calculated based on the priority of resources running on the node
	pe_working_set_t *data_set; //!< Cluster that this node is part of
	};

	struct pe_node_s {
	int weight;
	gboolean fixed; //!< \deprecated Will be removed in a future release
	int count;
	struct pe_node_shared_s *details;
	int rsc_discover_mode;
	};

	# define pe_rsc_orphan 0x00000001ULL
	# define pe_rsc_managed 0x00000002ULL
	# define pe_rsc_block 0x00000004ULL
	# define pe_rsc_orphan_container_filler 0x00000008ULL

	# define pe_rsc_notify 0x00000010ULL
	# define pe_rsc_unique 0x00000020ULL
	# define pe_rsc_fence_device 0x00000040ULL
	# define pe_rsc_promotable 0x00000080ULL

	# define pe_rsc_provisional 0x00000100ULL
	# define pe_rsc_allocating 0x00000200ULL
	# define pe_rsc_merging 0x00000400ULL
	# define pe_rsc_restarting 0x00000800ULL

	# define pe_rsc_stop 0x00001000ULL
	# define pe_rsc_reload 0x00002000ULL
	# define pe_rsc_allow_remote_remotes 0x00004000ULL
	# define pe_rsc_critical 0x00008000ULL

	# define pe_rsc_failed 0x00010000ULL
	# define pe_rsc_detect_loop 0x00020000ULL
	# define pe_rsc_runnable 0x00040000ULL
	# define pe_rsc_start_pending 0x00080000ULL

	//!< \deprecated Do not use
	# define pe_rsc_starting 0x00100000ULL

	//!< \deprecated Do not use
	# define pe_rsc_stopping 0x00200000ULL

	# define pe_rsc_stop_unexpected 0x00400000ULL
	# define pe_rsc_allow_migrate 0x00800000ULL

	# define pe_rsc_failure_ignored 0x01000000ULL
	# define pe_rsc_replica_container 0x02000000ULL
	# define pe_rsc_maintenance 0x04000000ULL
	# define pe_rsc_is_container 0x08000000ULL

	# define pe_rsc_needs_quorum 0x10000000ULL
	# define pe_rsc_needs_fencing 0x20000000ULL
	# define pe_rsc_needs_unfencing 0x40000000ULL

	/* INDENT-OFF */
	enum pe_action_flags {
	pe_action_pseudo = 0x00001,
	pe_action_runnable = 0x00002,
	pe_action_optional = 0x00004,
	pe_action_print_always = 0x00008,

	pe_action_have_node_attrs = 0x00010,
	pe_action_implied_by_stonith = 0x00040,
	pe_action_migrate_runnable = 0x00080,

	pe_action_dumped = 0x00100,
	pe_action_processed = 0x00200,
	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	pe_action_clear = 0x00400, //! \deprecated Unused
	#endif
	pe_action_dangle = 0x00800,

	/* This action requires one or more of its dependencies to be runnable.
	* We use this to clear the runnable flag before checking dependencies.
	*/
	pe_action_requires_any = 0x01000,

	pe_action_reschedule = 0x02000,
	pe_action_tracking = 0x04000,
	pe_action_dedup = 0x08000, //! Internal state tracking when creating graph

	pe_action_dc = 0x10000, //! Action may run on DC instead of target
	};
	/* INDENT-ON */

	struct pe_resource_s {
	char *id;
	char *clone_name;
	xmlNode *xml;
	xmlNode *orig_xml;
	xmlNode *ops_xml;

	pe_working_set_t *cluster;
	pe_resource_t *parent;

	enum pe_obj_types variant;
	void *variant_opaque;
	resource_object_functions_t *fns;
	resource_alloc_functions_t *cmds;

	enum rsc_recovery_type recovery_type;

	enum pe_restart restart_type; //!< \deprecated will be removed in future release

	int priority;
	int stickiness;
	int sort_index;
	int failure_timeout;
	int migration_threshold;
	guint remote_reconnect_ms;
	char *pending_task;

	unsigned long long flags;

	// @TODO merge these into flags
	gboolean is_remote_node;
	gboolean exclusive_discover;

	/* Pay special attention to whether you want to use rsc_cons_lhs and
	* rsc_cons directly, which include only colocations explicitly involving
	* this resource, or call libpacemaker's pcmk__with_this_colocations() and
	* pcmk__this_with_colocations() functions, which may return relevant
	* colocations involving the resource's ancestors as well.
	*/

	//!@{
	//! This field should be treated as internal to Pacemaker
	GList rsc_cons_lhs; // List of pcmk__colocation_t
	GList rsc_cons; // List of pcmk__colocation_t
	GList rsc_location; // List of pe__location_t
	GList actions; // List of pe_action_t
	GList rsc_tickets; // List of rsc_ticket
	//!@}

	pe_node_t *allocated_to;
	pe_node_t *partial_migration_target;
	pe_node_t *partial_migration_source;
	GList running_on; / pe_node_t* */
	GHashTable known_on; / pe_node_t* */
	GHashTable allowed_nodes; / pe_node_t* */

	enum rsc_role_e role;
	enum rsc_role_e next_role;

	GHashTable *meta;
	GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
	GHashTable *utilization;

	GList children; / pe_resource_t* */
	GList dangling_migrations; / pe_node_t* */

	pe_resource_t *container;
	GList *fillers;

	// @COMPAT These should be made const at next API compatibility break
	pe_node_t *pending_node; // Node on which pending_task is happening
	pe_node_t *lock_node; // Resource is shutdown-locked to this node

	time_t lock_time; // When shutdown lock started

	/* Resource parameters may have node-attribute-based rules, which means the
	* values can vary by node. This table is a cache of parameter name/value
	* tables for each node (as needed). Use pe_rsc_params() to get the table
	* for a given node.
	*/
	GHashTable *parameter_cache; // Key = node name, value = parameters table
	};

	struct pe_action_s {
	int id;
	int priority;

	pe_resource_t *rsc;
	pe_node_t *node;
	xmlNode *op_entry;

	char *task;
	char *uuid;
	char *cancel_task;
	char *reason;

	enum pe_action_flags flags;
	enum rsc_start_requirement needs;
	enum action_fail_response on_fail;
	enum rsc_role_e fail_role;

	GHashTable *meta;
	GHashTable *extra;

	/*
	* These two varables are associated with the constraint logic
	* that involves first having one or more actions runnable before
	* then allowing this action to execute.
	*
	* These varables are used with features such as 'clone-min' which
	* requires at minimum X number of cloned instances to be running
	* before an order dependency can run. Another option that uses
	* this is 'require-all=false' in ordering constrants. This option
	* says "only require one instance of a resource to start before
	* allowing dependencies to start" -- basically, require-all=false is
	* the same as clone-min=1.
	*/

	/* current number of known runnable actions in the before list. */
	int runnable_before;
	/* the number of "before" runnable actions required for this action
	* to be considered runnable */
	int required_runnable_before;

	GList actions_before; / pe_action_wrapper_t* */
	GList actions_after; / pe_action_wrapper_t* */

	/* Some of the above fields could be moved to the details,
	* except for API backward compatibility.
	*/
	void *action_details; // varies by type of action
	};

	typedef struct pe_ticket_s {
	char *id;
	gboolean granted;
	time_t last_granted;
	gboolean standby;
	GHashTable *state;
	} pe_ticket_t;

	typedef struct pe_tag_s {
	char *id;
	GList *refs;
	} pe_tag_t;

	//! Internal tracking for transition graph creation
	enum pe_link_state {
	pe_link_not_dumped, //! Internal tracking for transition graph creation
	pe_link_dumped, //! Internal tracking for transition graph creation
	pe_link_dup, //! \deprecated No longer used by Pacemaker
	};

	enum pe_discover_e {
	pe_discover_always = 0,
	pe_discover_never,
	pe_discover_exclusive,
	};

	/* INDENT-OFF */
	enum pe_ordering {
	pe_order_none = 0x0, /* deleted */
	pe_order_optional = 0x1, /* pure ordering, nothing implied */
	pe_order_apply_first_non_migratable = 0x2, /* Only apply this constraint's ordering if first is not migratable. */

	pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */
	pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */
	pe_order_promoted_implies_first = 0x40, /* If 'then' is required and then's rsc is promoted, ensure 'first' becomes required too */

	/* first requires then to be both runnable and migrate runnable. */
	pe_order_implies_first_migratable = 0x80,

	pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */

	pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */
	pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX',
	* ensure instances of 'then' on 'nodeX' are too.
	* Only really useful if 'then' is a clone and 'first' is not
	*/
	pe_order_probe = 0x800, /* If 'first->rsc' is
	* - running but about to stop, ignore the constraint
	* - otherwise, behave as runnable_left
	*/

	pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */
	pe_order_stonith_stop = 0x2000, //<! \deprecated Will be removed in future release
	pe_order_serialize_only = 0x4000, /* serialize */
	pe_order_same_node = 0x8000, /* applies only if 'first' and 'then' are on same node */

	pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
	pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not mandatory */

	pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */
	pe_order_load = 0x200000, /* Only relevant if... */
	pe_order_one_or_more = 0x400000, /* 'then' is runnable only if one or more of its dependencies are too */
	pe_order_anti_colocation = 0x800000,

	pe_order_preserve = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
	pe_order_then_cancels_first = 0x2000000, // if 'then' becomes required, 'first' becomes optional
	pe_order_trace = 0x4000000, /* test marker */

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	// \deprecated Use pe_order_promoted_implies_first instead
	pe_order_implies_first_master = pe_order_promoted_implies_first,
	#endif
	};
	/* INDENT-ON */

	typedef struct pe_action_wrapper_s {
	enum pe_ordering type;
	enum pe_link_state state;
	pe_action_t *action;
	} pe_action_wrapper_t;

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/pengine/pe_types_compat.h>
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_PENGINE_PE_TYPES__H
	diff --git a/include/crm/pengine/pe_types_compat.h b/include/crm/pengine/pe_types_compat.h
	index 566a5bf5b6..8d253155e0 100644
	--- a/include/crm/pengine/pe_types_compat.h
	+++ b/include/crm/pengine/pe_types_compat.h
	@@ -1,140 +1,143 @@
	/*
	* Copyright 2004-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	# define PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H

	#include <crm/pengine/pe_types.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker scheduler API
	* \ingroup pengine
	* \deprecated Do not include this header directly. The scheduler APIs in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	//! \deprecated Use pcmk_sched_quorate instead
	#define pe_flag_have_quorum pcmk_sched_quorate

	//! \deprecated Use pcmk_sched_symmetric_cluster instead
	#define pe_flag_symmetric_cluster pcmk_sched_symmetric_cluster

	//! \deprecated Use pcmk_sched_in_maintenance instead
	#define pe_flag_maintenance_mode pcmk_sched_in_maintenance

	//! \deprecated Use pcmk_sched_fencing_enabled instead
	#define pe_flag_stonith_enabled pcmk_sched_fencing_enabled

	//! \deprecated Use pcmk_sched_have_fencing instead
	#define pe_flag_have_stonith_resource pcmk_sched_have_fencing

	//! \deprecated Use pcmk_sched_enable_unfencing instead
	#define pe_flag_enable_unfencing pcmk_sched_enable_unfencing

	//! \deprecated Use pcmk_sched_concurrent_fencing instead
	#define pe_flag_concurrent_fencing pcmk_sched_concurrent_fencing

	//! \deprecated Use pcmk_sched_stop_removed_resources instead
	#define pe_flag_stop_rsc_orphans pcmk_sched_stop_removed_resources

	//! \deprecated Use pcmk_sched_cancel_removed_actions instead
	#define pe_flag_stop_action_orphans pcmk_sched_cancel_removed_actions

	//! \deprecated Use pcmk_sched_stop_all instead
	#define pe_flag_stop_everything pcmk_sched_stop_all

	//! \deprecated Use pcmk_sched_start_failure_fatal instead
	#define pe_flag_start_failure_fatal pcmk_sched_start_failure_fatal

	//! \deprecated Do not use
	#define pe_flag_remove_after_stop pcmk_sched_remove_after_stop

	//! \deprecated Use pcmk_sched_startup_fencing instead
	#define pe_flag_startup_fencing pcmk_sched_startup_fencing

	//! \deprecated Use pcmk_sched_shutdown_lock instead
	#define pe_flag_shutdown_lock pcmk_sched_shutdown_lock

	//! \deprecated Use pcmk_sched_probe_resources instead
	#define pe_flag_startup_probes pcmk_sched_probe_resources

	//! \deprecated Use pcmk_sched_have_status instead
	#define pe_flag_have_status pcmk_sched_have_status

	//! \deprecated Use pcmk_sched_have_remote_nodes instead
	#define pe_flag_have_remote_nodes pcmk_sched_have_remote_nodes

	//! \deprecated Use pcmk_sched_location_only instead
	#define pe_flag_quick_location pcmk_sched_location_only

	//! \deprecated Use pcmk_sched_sanitized instead
	#define pe_flag_sanitized pcmk_sched_sanitized

	//! \deprecated Do not use
	#define pe_flag_stdout (1ULL << 22)

	//! \deprecated Use pcmk_sched_no_counts instead
	#define pe_flag_no_counts pcmk_sched_no_counts

	//! \deprecated Use pcmk_sched_no_compat instead
	#define pe_flag_no_compat pcmk_sched_no_compat

	//! \deprecated Use pcmk_sched_output_scores instead
	#define pe_flag_show_scores pcmk_sched_output_scores

	+//! \deprecated Use pcmk_sched_show_utilization instead
	+#define pe_flag_show_utilization pcmk_sched_show_utilization
	+
	//!@{
	//! \deprecated Do not use (unused by Pacemaker)
	enum pe_graph_flags {
	pe_graph_none = 0x00000,
	pe_graph_updated_first = 0x00001,
	pe_graph_updated_then = 0x00002,
	pe_graph_disable = 0x00004,
	};
	//!@}

	//!@{
	//! \deprecated Do not use
	enum pe_check_parameters {
	pe_check_last_failure,
	pe_check_active,
	};
	//!@}

	//!< \deprecated Use pe_action_t instead
	typedef struct pe_action_s action_t;

	//!< \deprecated Use pe_action_wrapper_t instead
	typedef struct pe_action_wrapper_s action_wrapper_t;

	//!< \deprecated Use pe_node_t instead
	typedef struct pe_node_s node_t;

	//!< \deprecated Use enum pe_quorum_policy instead
	typedef enum pe_quorum_policy no_quorum_policy_t;

	//!< \deprecated use pe_resource_t instead
	typedef struct pe_resource_s resource_t;

	//!< \deprecated Use pe_tag_t instead
	typedef struct pe_tag_s tag_t;

	//!< \deprecated Use pe_ticket_t instead
	typedef struct pe_ticket_s ticket_t;

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
	diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
	index e1db9b7670..d98e3fefee 100644
	--- a/lib/pacemaker/pcmk_sched_resource.c
	+++ b/lib/pacemaker/pcmk_sched_resource.c
	@@ -1,771 +1,771 @@
	/*
	* Copyright 2014-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <string.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Resource assignment methods by resource variant
	static resource_alloc_functions_t assignment_methods[] = {
	{
	pcmk__primitive_assign,
	pcmk__primitive_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__primitive_internal_constraints,
	pcmk__primitive_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_primitive_colocations,
	pcmk__primitive_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__apply_location,
	pcmk__primitive_action_flags,
	pcmk__update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__primitive_add_graph_meta,
	pcmk__primitive_add_utilization,
	pcmk__primitive_shutdown_lock,
	},
	{
	pcmk__group_assign,
	pcmk__group_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__group_internal_constraints,
	pcmk__group_apply_coloc_score,
	pcmk__group_colocated_resources,
	pcmk__with_group_colocations,
	pcmk__group_with_colocations,
	pcmk__group_add_colocated_node_scores,
	pcmk__group_apply_location,
	pcmk__group_action_flags,
	pcmk__group_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__group_add_utilization,
	pcmk__group_shutdown_lock,
	},
	{
	pcmk__clone_assign,
	pcmk__clone_create_actions,
	pcmk__clone_create_probe,
	pcmk__clone_internal_constraints,
	pcmk__clone_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_clone_colocations,
	pcmk__clone_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__clone_apply_location,
	pcmk__clone_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__clone_add_actions_to_graph,
	pcmk__clone_add_graph_meta,
	pcmk__clone_add_utilization,
	pcmk__clone_shutdown_lock,
	},
	{
	pcmk__bundle_assign,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_bundle_colocations,
	pcmk__bundle_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__bundle_apply_location,
	pcmk__bundle_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_bundle_actions,
	pcmk__bundle_add_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__bundle_add_utilization,
	pcmk__bundle_shutdown_lock,
	}
	};

	/*!
	* \internal
	* \brief Check whether a resource's agent standard, provider, or type changed
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] node Node needing unfencing if agent changed
	* \param[in] rsc_entry XML with previously known agent information
	* \param[in] active_on_node Whether \p rsc is active on \p node
	*
	* \return true if agent for \p rsc changed, otherwise false
	*/
	bool
	pcmk__rsc_agent_changed(pe_resource_t rsc, pe_node_t node,
	const xmlNode *rsc_entry, bool active_on_node)
	{
	bool changed = false;
	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
	const char *value = crm_element_value(rsc->xml, attr_list[i]);
	const char *old_value = crm_element_value(rsc_entry, attr_list[i]);

	if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
	changed = true;
	trigger_unfencing(rsc, node, "Device definition changed", NULL,
	rsc->cluster);
	if (active_on_node) {
	crm_notice("Forcing restart of %s on %s "
	"because %s changed from '%s' to '%s'",
	rsc->id, pe__node_name(node), attr_list[i],
	pcmk__s(old_value, ""), pcmk__s(value, ""));
	}
	}
	}
	if (changed && active_on_node) {
	// Make sure the resource is restarted
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE, TRUE,
	rsc->cluster);
	pe__set_resource_flags(rsc, pe_rsc_start_pending);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Add resource (and any matching children) to list if it matches ID
	*
	* \param[in] result List to add resource to
	* \param[in] rsc Resource to check
	* \param[in] id ID to match
	*
	* \return (Possibly new) head of list
	*/
	static GList *
	add_rsc_if_matching(GList result, pe_resource_t rsc, const char *id)
	{
	if ((strcmp(rsc->id, id) == 0)
	\|\| ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) {
	result = g_list_prepend(result, rsc);
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	result = add_rsc_if_matching(result, child, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Find all resources matching a given ID by either ID or clone name
	*
	* \param[in] id Resource ID to check
	* \param[in] data_set Cluster working set
	*
	* \return List of all resources that match \p id
	* \note The caller is responsible for freeing the return value with
	* g_list_free().
	*/
	GList *
	pcmk__rscs_matching_id(const char id, const pe_working_set_t data_set)
	{
	GList *result = NULL;

	CRM_CHECK((id != NULL) && (data_set != NULL), return NULL);
	for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
	result = add_rsc_if_matching(result, (pe_resource_t *) iter->data, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for a resource
	*
	* \param[in,out] data Resource to set assignment methods for
	* \param[in] user_data Ignored
	*/
	static void
	set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
	{
	pe_resource_t *rsc = data;

	rsc->cmds = &assignment_methods[rsc->variant];
	g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for all resources
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__set_assignment_methods(pe_working_set_t *data_set)
	{
	g_list_foreach(data_set->resources, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Wrapper for colocated_resources() method for readability
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] list Pointer to list to add to
	*
	* \return (Possibly new) head of list
	*/
	static inline void
	add_colocated_resources(const pe_resource_t rsc, const pe_resource_t orig_rsc,
	GList **list)
	{
	list = rsc->cmds->colocated_resources(rsc, orig_rsc, list);
	}

	// Shared implementation of resource_alloc_functions_t:colocated_resources()
	GList *
	pcmk__colocated_resources(const pe_resource_t *rsc,
	const pe_resource_t orig_rsc, GList colocated_rscs)
	{
	const GList *iter = NULL;
	GList *colocations = NULL;

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if ((rsc == NULL) \|\| (g_list_find(colocated_rscs, rsc) != NULL)) {
	return colocated_rscs;
	}

	pe_rsc_trace(orig_rsc, "%s is in colocation chain with %s",
	rsc->id, orig_rsc->id);
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);

	// Follow colocations where this resource is the dependent resource
	colocations = pcmk__this_with_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pe_resource_t *primary = constraint->primary;

	if (primary == orig_rsc) {
	continue; // Break colocation loop
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(rsc, primary, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(primary, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	// Follow colocations where this resource is the primary resource
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pe_resource_t *dependent = constraint->dependent;

	if (dependent == orig_rsc) {
	continue; // Break colocation loop
	}

	if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) {
	continue; // We can't be sure whether dependent will be colocated
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(dependent, rsc, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	return colocated_rscs;
	}

	// No-op function for variants that don't need to implement add_graph_meta()
	void
	pcmk__noop_add_graph_meta(const pe_resource_t rsc, xmlNode xml)
	{
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void
	pcmk__output_resource_actions(pe_resource_t *rsc)
	{
	pe_node_t *next = NULL;
	pe_node_t *current = NULL;
	pcmk__output_t *out = NULL;

	CRM_ASSERT(rsc != NULL);

	out = rsc->cluster->priv;
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	child->cmds->output_actions(child);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pe__current_node(rsc);
	if (rsc->role == pcmk_role_stopped) {
	/* This can occur when resources are being recovered because
	* the current role can change in pcmk__primitive_create_actions()
	*/
	rsc->role = pcmk_role_started;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next);
	}

	/*!
	* \internal
	* \brief Add a resource to a node's list of assigned resources
	*
	* \param[in,out] node Node to add resource to
	* \param[in] rsc Resource to add
	*/
	static inline void
	add_assigned_resource(pe_node_t node, pe_resource_t rsc)
	{
	node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc,
	rsc);
	}

	/*!
	* \internal
	* \brief Assign a specified resource (of any variant) to a node
	*
	* Assign a specified resource and its children (if any) to a specified node, if
	* the node can run the resource (or unconditionally, if \p force is true). Mark
	* the resources as no longer provisional.
	*
	* If a resource can't be assigned (or \p node is \c NULL), unassign any
	* previous assignment. If \p stop_if_fail is \c true, set next role to stopped
	* and update any existing actions scheduled for the resource.
	*
	* \param[in,out] rsc Resource to assign
	* \param[in,out] node Node to assign \p rsc to
	* \param[in] force If true, assign to \p node even if unavailable
	* \param[in] stop_if_fail If \c true and either \p rsc can't be assigned
	* or \p chosen is \c NULL, set next role to
	* stopped and update existing actions (if \p rsc
	* is not a primitive, this applies to its
	* primitive descendants instead)
	*
	* \return \c true if the assignment of \p rsc changed, or \c false otherwise
	*
	* \note Assigning a resource to the NULL node using this function is different
	* from calling pcmk__unassign_resource(), in that it may also update any
	* actions created for the resource.
	* \note The \c resource_alloc_functions_t:assign() method is preferred, unless
	* a resource should be assigned to the \c NULL node or every resource in
	* a tree should be assigned to the same node.
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	bool
	pcmk__assign_resource(pe_resource_t rsc, pe_node_t node, bool force,
	bool stop_if_fail)
	{
	bool changed = false;

	CRM_ASSERT(rsc != NULL);

	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t *child_rsc = iter->data;

	changed \|= pcmk__assign_resource(child_rsc, node, force,
	stop_if_fail);
	}
	return changed;
	}

	// Assigning a primitive

	if (!force && (node != NULL)
	&& ((node->weight < 0)
	// Allow graph to assume that guest node connections will come up
	\|\| (!pcmk__node_available(node, true, false)
	&& !pe__is_guest_node(node)))) {

	pe_rsc_debug(rsc,
	"All nodes for resource %s are unavailable, unclean or "
	"shutting down (%s can%s run resources, with score %s)",
	rsc->id, pe__node_name(node),
	(pcmk__node_available(node, true, false)? "" : "not"),
	pcmk_readable_score(node->weight));

	if (stop_if_fail) {
	pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
	}
	node = NULL;
	}

	if (rsc->allocated_to != NULL) {
	changed = !pe__same_node(rsc->allocated_to, node);
	} else {
	changed = (node != NULL);
	}
	pcmk__unassign_resource(rsc);
	pe__clear_resource_flags(rsc, pe_rsc_provisional);

	if (node == NULL) {
	char *rc_stopped = NULL;

	pe_rsc_debug(rsc, "Could not assign %s to a node", rsc->id);

	if (!stop_if_fail) {
	return changed;
	}
	pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");

	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pe_action_t op = (pe_action_t ) iter->data;

	pe_rsc_debug(rsc, "Updating %s for %s assignment failure",
	op->uuid, rsc->id);

	if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	pe__clear_action_flags(op, pe_action_optional);

	} else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	pe__clear_action_flags(op, pe_action_runnable);

	} else {
	// Cancel recurring actions, unless for stopped state
	const char *interval_ms_s = NULL;
	const char *target_rc_s = NULL;

	interval_ms_s = g_hash_table_lookup(op->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	target_rc_s = g_hash_table_lookup(op->meta,
	XML_ATTR_TE_TARGET_RC);
	if (rc_stopped == NULL) {
	rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	}

	if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	&& !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {

	pe__clear_action_flags(op, pe_action_runnable);
	}
	}
	}
	free(rc_stopped);
	return changed;
	}

	pe_rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node));
	rsc->allocated_to = pe__copy_node(node);

	add_assigned_resource(node, rsc);
	node->details->num_resources++;
	node->count++;
	pcmk__consume_node_capacity(node->details->utilization, rsc);

	- if (pcmk_is_set(rsc->cluster->flags, pe_flag_show_utilization)) {
	+ if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->message(out, "resource-util", rsc, node, __func__);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Remove any node assignment from a specified resource and its children
	*
	* If a specified resource has been assigned to a node, remove that assignment
	* and mark the resource as provisional again.
	*
	* \param[in,out] rsc Resource to unassign
	*
	* \note This function is called recursively on \p rsc and its children.
	*/
	void
	pcmk__unassign_resource(pe_resource_t *rsc)
	{
	pe_node_t *old = rsc->allocated_to;

	if (old == NULL) {
	crm_info("Unassigning %s", rsc->id);
	} else {
	crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old));
	}

	pe__set_resource_flags(rsc, pe_rsc_provisional);

	if (rsc->children == NULL) {
	if (old == NULL) {
	return;
	}
	rsc->allocated_to = NULL;

	/* We're going to free the pe_node_t, but its details member is shared
	* and will remain, so update that appropriately first.
	*/
	old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc,
	rsc);
	old->details->num_resources--;
	pcmk__release_node_capacity(old->details->utilization, rsc);
	free(old);
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__unassign_resource((pe_resource_t *) iter->data);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has reached its migration threshold on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to check
	* \param[out] failed If threshold has been reached, this will be set to
	* resource that failed (possibly a parent of \p rsc)
	*
	* \return true if the migration threshold has been reached, false otherwise
	*/
	bool
	pcmk__threshold_reached(pe_resource_t rsc, const pe_node_t node,
	pe_resource_t **failed)
	{
	int fail_count, remaining_tries;
	pe_resource_t *rsc_to_ban = rsc;

	// Migration threshold of 0 means never force away
	if (rsc->migration_threshold == 0) {
	return false;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	return false;
	}

	// If there are no failures, there's no need to force away
	fail_count = pe_get_failcount(node, rsc, NULL,
	pe_fc_effective\|pe_fc_fillers, NULL);
	if (fail_count <= 0) {
	return false;
	}

	// If failed resource is anonymous clone instance, we'll force clone away
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	rsc_to_ban = uber_parent(rsc);
	}

	// How many more times recovery will be tried on this node
	remaining_tries = rsc->migration_threshold - fail_count;

	if (remaining_tries <= 0) {
	crm_warn("%s cannot run on %s due to reaching migration threshold "
	"(clean up resource to allow again)"
	CRM_XS " failures=%d migration-threshold=%d",
	rsc_to_ban->id, pe__node_name(node), fail_count,
	rsc->migration_threshold);
	if (failed != NULL) {
	*failed = rsc_to_ban;
	}
	return true;
	}

	crm_info("%s can fail %d more time%s on "
	"%s before reaching migration threshold (%d)",
	rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
	pe__node_name(node), rsc->migration_threshold);
	return false;
	}

	/*!
	* \internal
	* \brief Get a node's score
	*
	* \param[in] node Node with ID to check
	* \param[in] nodes List of nodes to look for \p node score in
	*
	* \return Node's score, or -INFINITY if not found
	*/
	static int
	get_node_score(const pe_node_t node, GHashTable nodes)
	{
	pe_node_t *found_node = NULL;

	if ((node != NULL) && (nodes != NULL)) {
	found_node = g_hash_table_lookup(nodes, node->details->id);
	}
	return (found_node == NULL)? -INFINITY : found_node->weight;
	}

	/*!
	* \internal
	* \brief Compare two resources according to which should be assigned first
	*
	* \param[in] a First resource to compare
	* \param[in] b Second resource to compare
	* \param[in] data Sorted list of all nodes in cluster
	*
	* \return -1 if \p a should be assigned before \b, 0 if they are equal,
	* or +1 if \p a should be assigned after \b
	*/
	static gint
	cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
	{
	/* GLib insists that this function require gconstpointer arguments, but we
	* make a small, temporary change to each argument (setting the
	* pe_rsc_merging flag) during comparison
	*/
	pe_resource_t resource1 = (pe_resource_t ) a;
	pe_resource_t resource2 = (pe_resource_t ) b;
	const GList *nodes = data;

	int rc = 0;
	int r1_score = -INFINITY;
	int r2_score = -INFINITY;
	pe_node_t *r1_node = NULL;
	pe_node_t *r2_node = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;
	const char *reason = NULL;

	// Resources with highest priority should be assigned first
	reason = "priority";
	r1_score = resource1->priority;
	r2_score = resource2->priority;
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// We need nodes to make any other useful comparisons
	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	// Calculate and log node scores
	resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id,
	&r1_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id,
	&r2_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);
	pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	// The resource with highest score on its current node goes first
	reason = "current location";
	if (resource1->running_on != NULL) {
	r1_node = pe__current_node(resource1);
	}
	if (resource2->running_on != NULL) {
	r2_node = pe__current_node(resource2);
	}
	r1_score = get_node_score(r1_node, r1_nodes);
	r2_score = get_node_score(r2_node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// Otherwise a higher score on any node will do
	reason = "score";
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	const pe_node_t node = (const pe_node_t ) iter->data;

	r1_score = get_node_score(node, r1_nodes);
	r2_score = get_node_score(node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
	resource1->id, r1_score,
	((r1_node == NULL)? "" : " on "),
	((r1_node == NULL)? "" : r1_node->details->id),
	((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
	resource2->id, r2_score,
	((r2_node == NULL)? "" : " on "),
	((r2_node == NULL)? "" : r2_node->details->id),
	reason);
	if (r1_nodes != NULL) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes != NULL) {
	g_hash_table_destroy(r2_nodes);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Sort resources in the order they should be assigned to nodes
	*
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__sort_resources(pe_working_set_t *data_set)
	{
	GList *nodes = g_list_copy(data_set->nodes);

	nodes = pcmk__sort_nodes(nodes, NULL);
	data_set->resources = g_list_sort_with_data(data_set->resources,
	cmp_resources, nodes);
	g_list_free(nodes);
	}
	diff --git a/lib/pacemaker/pcmk_sched_utilization.c b/lib/pacemaker/pcmk_sched_utilization.c
	index 8297d429be..ee0563ed94 100644
	--- a/lib/pacemaker/pcmk_sched_utilization.c
	+++ b/lib/pacemaker/pcmk_sched_utilization.c
	@@ -1,465 +1,465 @@
	/*
	* Copyright 2014-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get integer utilization from a string
	*
	* \param[in] s String representation of a node utilization value
	*
	* \return Integer equivalent of \p s
	* \todo It would make sense to restrict utilization values to nonnegative
	* integers, but the documentation just says "integers" and we didn't
	* restrict them initially, so for backward compatibility, allow any
	* integer.
	*/
	static int
	utilization_value(const char *s)
	{
	int value = 0;

	if ((s != NULL) && (pcmk__scan_min_int(s, &value, INT_MIN) == EINVAL)) {
	pe_warn("Using 0 for utilization instead of invalid value '%s'", value);
	value = 0;
	}
	return value;
	}


	/*
	* Functions for comparing node capacities
	*/

	struct compare_data {
	const pe_node_t *node1;
	const pe_node_t *node2;
	bool node2_only;
	int result;
	};

	/*!
	* \internal
	* \brief Compare a single utilization attribute for two nodes
	*
	* Compare one utilization attribute for two nodes, incrementing the result if
	* the first node has greater capacity, and decrementing it if the second node
	* has greater capacity.
	*
	* \param[in] key Utilization attribute name to compare
	* \param[in] value Utilization attribute value to compare
	* \param[in,out] user_data Comparison data (as struct compare_data*)
	*/
	static void
	compare_utilization_value(gpointer key, gpointer value, gpointer user_data)
	{
	int node1_capacity = 0;
	int node2_capacity = 0;
	struct compare_data *data = user_data;
	const char *node2_value = NULL;

	if (data->node2_only) {
	if (g_hash_table_lookup(data->node1->details->utilization, key)) {
	return; // We've already compared this attribute
	}
	} else {
	node1_capacity = utilization_value((const char *) value);
	}

	node2_value = g_hash_table_lookup(data->node2->details->utilization, key);
	node2_capacity = utilization_value(node2_value);

	if (node1_capacity > node2_capacity) {
	data->result--;
	} else if (node1_capacity < node2_capacity) {
	data->result++;
	}
	}

	/*!
	* \internal
	* \brief Compare utilization capacities of two nodes
	*
	* \param[in] node1 First node to compare
	* \param[in] node2 Second node to compare
	*
	* \return Negative integer if node1 has more free capacity,
	* 0 if the capacities are equal, or a positive integer
	* if node2 has more free capacity
	*/
	int
	pcmk__compare_node_capacities(const pe_node_t node1, const pe_node_t node2)
	{
	struct compare_data data = {
	.node1 = node1,
	.node2 = node2,
	.node2_only = false,
	.result = 0,
	};

	// Compare utilization values that node1 and maybe node2 have
	g_hash_table_foreach(node1->details->utilization, compare_utilization_value,
	&data);

	// Compare utilization values that only node2 has
	data.node2_only = true;
	g_hash_table_foreach(node2->details->utilization, compare_utilization_value,
	&data);

	return data.result;
	}


	/*
	* Functions for updating node capacities
	*/

	struct calculate_data {
	GHashTable *current_utilization;
	bool plus;
	};

	/*!
	* \internal
	* \brief Update a single utilization attribute with a new value
	*
	* \param[in] key Name of utilization attribute to update
	* \param[in] value Value to add or substract
	* \param[in,out] user_data Calculation data (as struct calculate_data *)
	*/
	static void
	update_utilization_value(gpointer key, gpointer value, gpointer user_data)
	{
	int result = 0;
	const char *current = NULL;
	struct calculate_data *data = user_data;

	current = g_hash_table_lookup(data->current_utilization, key);
	if (data->plus) {
	result = utilization_value(current) + utilization_value(value);
	} else if (current) {
	result = utilization_value(current) - utilization_value(value);
	}
	g_hash_table_replace(data->current_utilization,
	strdup(key), pcmk__itoa(result));
	}

	/*!
	* \internal
	* \brief Subtract a resource's utilization from node capacity
	*
	* \param[in,out] current_utilization Current node utilization attributes
	* \param[in] rsc Resource with utilization to subtract
	*/
	void
	pcmk__consume_node_capacity(GHashTable *current_utilization,
	const pe_resource_t *rsc)
	{
	struct calculate_data data = {
	.current_utilization = current_utilization,
	.plus = false,
	};

	g_hash_table_foreach(rsc->utilization, update_utilization_value, &data);
	}

	/*!
	* \internal
	* \brief Add a resource's utilization to node capacity
	*
	* \param[in,out] current_utilization Current node utilization attributes
	* \param[in] rsc Resource with utilization to add
	*/
	void
	pcmk__release_node_capacity(GHashTable *current_utilization,
	const pe_resource_t *rsc)
	{
	struct calculate_data data = {
	.current_utilization = current_utilization,
	.plus = true,
	};

	g_hash_table_foreach(rsc->utilization, update_utilization_value, &data);
	}


	/*
	* Functions for checking for sufficient node capacity
	*/

	struct capacity_data {
	const pe_node_t *node;
	const char *rsc_id;
	bool is_enough;
	};

	/*!
	* \internal
	* \brief Check whether a single utilization attribute has sufficient capacity
	*
	* \param[in] key Name of utilization attribute to check
	* \param[in] value Amount of utilization required
	* \param[in,out] user_data Capacity data (as struct capacity_data *)
	*/
	static void
	check_capacity(gpointer key, gpointer value, gpointer user_data)
	{
	int required = 0;
	int remaining = 0;
	const char *node_value_s = NULL;
	struct capacity_data *data = user_data;

	node_value_s = g_hash_table_lookup(data->node->details->utilization, key);

	required = utilization_value(value);
	remaining = utilization_value(node_value_s);

	if (required > remaining) {
	crm_debug("Remaining capacity for %s on %s (%d) is insufficient "
	"for resource %s usage (%d)",
	(const char *) key, pe__node_name(data->node), remaining,
	data->rsc_id, required);
	data->is_enough = false;
	}
	}

	/*!
	* \internal
	* \brief Check whether a node has sufficient capacity for a resource
	*
	* \param[in] node Node to check
	* \param[in] rsc_id ID of resource to check (for debug logs only)
	* \param[in] utilization Required utilization amounts
	*
	* \return true if node has sufficient capacity for resource, otherwise false
	*/
	static bool
	have_enough_capacity(const pe_node_t node, const char rsc_id,
	GHashTable *utilization)
	{
	struct capacity_data data = {
	.node = node,
	.rsc_id = rsc_id,
	.is_enough = true,
	};

	g_hash_table_foreach(utilization, check_capacity, &data);
	return data.is_enough;
	}

	/*!
	* \internal
	* \brief Sum the utilization requirements of a list of resources
	*
	* \param[in] orig_rsc Resource being assigned (for logging purposes)
	* \param[in] rscs Resources whose utilization should be summed
	*
	* \return Newly allocated hash table with sum of all utilization values
	* \note It is the caller's responsibility to free the return value using
	* g_hash_table_destroy().
	*/
	static GHashTable *
	sum_resource_utilization(const pe_resource_t orig_rsc, GList rscs)
	{
	GHashTable *utilization = pcmk__strkey_table(free, free);

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pe_resource_t rsc = (pe_resource_t ) iter->data;

	rsc->cmds->add_utilization(rsc, orig_rsc, rscs, utilization);
	}
	return utilization;
	}

	/*!
	* \internal
	* \brief Ban resource from nodes with insufficient utilization capacity
	*
	* \param[in,out] rsc Resource to check
	*
	* \return Allowed node for \p rsc with most spare capacity, if there are no
	* nodes with enough capacity for \p rsc and all its colocated resources
	*/
	const pe_node_t *
	pcmk__ban_insufficient_capacity(pe_resource_t *rsc)
	{
	bool any_capable = false;
	char *rscs_id = NULL;
	pe_node_t *node = NULL;
	const pe_node_t *most_capable_node = NULL;
	GList *colocated_rscs = NULL;
	GHashTable *unassigned_utilization = NULL;
	GHashTableIter iter;

	CRM_CHECK(rsc != NULL, return NULL);

	// The default placement strategy ignores utilization
	if (pcmk__str_eq(rsc->cluster->placement_strategy, "default",
	pcmk__str_casei)) {
	return NULL;
	}

	// Check whether any resources are colocated with this one
	colocated_rscs = rsc->cmds->colocated_resources(rsc, NULL, NULL);
	if (colocated_rscs == NULL) {
	return NULL;
	}

	rscs_id = crm_strdup_printf("%s and its colocated resources", rsc->id);

	// If rsc isn't in the list, add it so we include its utilization
	if (g_list_find(colocated_rscs, rsc) == NULL) {
	colocated_rscs = g_list_append(colocated_rscs, rsc);
	}

	// Sum utilization of colocated resources that haven't been assigned yet
	unassigned_utilization = sum_resource_utilization(rsc, colocated_rscs);

	// Check whether any node has enough capacity for all the resources
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (!pcmk__node_available(node, true, false)) {
	continue;
	}

	if (have_enough_capacity(node, rscs_id, unassigned_utilization)) {
	any_capable = true;
	}

	// Keep track of node with most free capacity
	if ((most_capable_node == NULL)
	\|\| (pcmk__compare_node_capacities(node, most_capable_node) < 0)) {
	most_capable_node = node;
	}
	}

	if (any_capable) {
	// If so, ban resource from any node with insufficient capacity
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)
	&& !have_enough_capacity(node, rscs_id,
	unassigned_utilization)) {
	pe_rsc_debug(rsc, "%s does not have enough capacity for %s",
	pe__node_name(node), rscs_id);
	resource_location(rsc, node, -INFINITY, "__limit_utilization__",
	rsc->cluster);
	}
	}
	most_capable_node = NULL;

	} else {
	// Otherwise, ban from nodes with insufficient capacity for rsc alone
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__node_available(node, true, false)
	&& !have_enough_capacity(node, rsc->id, rsc->utilization)) {
	pe_rsc_debug(rsc, "%s does not have enough capacity for %s",
	pe__node_name(node), rsc->id);
	resource_location(rsc, node, -INFINITY, "__limit_utilization__",
	rsc->cluster);
	}
	}
	}

	g_hash_table_destroy(unassigned_utilization);
	g_list_free(colocated_rscs);
	free(rscs_id);

	pe__show_node_scores(true, rsc, "Post-utilization", rsc->allowed_nodes,
	rsc->cluster);
	return most_capable_node;
	}

	/*!
	* \internal
	* \brief Create a new load_stopped pseudo-op for a node
	*
	* \param[in,out] node Node to create op for
	*
	* \return Newly created load_stopped op
	*/
	static pe_action_t *
	new_load_stopped_op(pe_node_t *node)
	{
	char *load_stopped_task = crm_strdup_printf(PCMK_ACTION_LOAD_STOPPED "_%s",
	node->details->uname);
	pe_action_t *load_stopped = get_pseudo_op(load_stopped_task,
	node->details->data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = pe__copy_node(node);
	pe__clear_action_flags(load_stopped, pe_action_optional);
	}
	free(load_stopped_task);
	return load_stopped;
	}

	/*!
	* \internal
	* \brief Create utilization-related internal constraints for a resource
	*
	* \param[in,out] rsc Resource to create constraints for
	* \param[in] allowed_nodes List of allowed next nodes for \p rsc
	*/
	void
	pcmk__create_utilization_constraints(pe_resource_t *rsc,
	const GList *allowed_nodes)
	{
	const GList *iter = NULL;
	pe_action_t *load_stopped = NULL;

	pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
	rsc->id, rsc->cluster->placement_strategy);

	// "stop rsc then load_stopped" constraints for current nodes
	for (iter = rsc->running_on; iter != NULL; iter = iter->next) {
	load_stopped = new_load_stopped_op(iter->data);
	pcmk__new_ordering(rsc, stop_key(rsc), NULL, NULL, NULL, load_stopped,
	pe_order_load, rsc->cluster);
	}

	// "load_stopped then start/migrate_to rsc" constraints for allowed nodes
	for (iter = allowed_nodes; iter; iter = iter->next) {
	load_stopped = new_load_stopped_op(iter->data);
	pcmk__new_ordering(NULL, NULL, load_stopped, rsc, start_key(rsc), NULL,
	pe_order_load, rsc->cluster);
	pcmk__new_ordering(NULL, NULL, load_stopped,
	rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MIGRATE_TO, 0),
	NULL,
	pe_order_load, rsc->cluster);
	}
	}

	/*!
	* \internal
	* \brief Output node capacities if enabled
	*
	* \param[in] desc Prefix for output
	* \param[in,out] data_set Cluster working set
	*/
	void
	pcmk__show_node_capacities(const char desc, pe_working_set_t data_set)
	{
	- if (!pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	+ if (!pcmk_is_set(data_set->flags, pcmk_sched_show_utilization)) {
	return;
	}
	for (const GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
	const pe_node_t node = (const pe_node_t ) iter->data;
	pcmk__output_t *out = data_set->priv;

	out->message(out, "node-capacity", node, desc);
	}
	}
	diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
	index fb7a4a55a4..0a1a62f5de 100644
	--- a/lib/pacemaker/pcmk_simulate.c
	+++ b/lib/pacemaker/pcmk_simulate.c
	@@ -1,1005 +1,1005 @@
	/*
	* Copyright 2021-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/pengine/pe_types.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <stdint.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	static pcmk__output_t *out = NULL;
	static cib_t *fake_cib = NULL;
	static GList *fake_resource_list = NULL;
	static const GList *fake_op_fail_list = NULL;

	static void set_effective_date(pe_working_set_t *data_set, bool print_original,
	const char *use_date);

	/*!
	* \internal
	* \brief Create an action name for use in a dot graph
	*
	* \param[in] action Action to create name for
	* \param[in] verbose If true, add action ID to name
	*
	* \return Newly allocated string with action name
	* \note It is the caller's responsibility to free the result.
	*/
	static char *
	create_action_name(const pe_action_t *action, bool verbose)
	{
	char *action_name = NULL;
	const char *prefix = "";
	const char *action_host = NULL;
	const char *clone_name = NULL;
	const char *task = action->task;

	if (action->node != NULL) {
	action_host = action->node->details->uname;
	} else if (!pcmk_is_set(action->flags, pe_action_pseudo)) {
	action_host = "<none>";
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	prefix = "Cancel ";
	task = action->cancel_task;
	}

	if (action->rsc != NULL) {
	clone_name = action->rsc->clone_name;
	}

	if (clone_name != NULL) {
	char *key = NULL;
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	const char *n_type = g_hash_table_lookup(action->meta,
	"notify_key_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_key_operation");

	CRM_ASSERT(n_type != NULL);
	CRM_ASSERT(n_task != NULL);
	key = pcmk__notify_key(clone_name, n_type, n_task);
	} else {
	key = pcmk__op_key(clone_name, task, interval_ms);
	}

	if (action_host != NULL) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, key, action_host);
	} else {
	action_name = crm_strdup_printf("%s%s", prefix, key);
	}
	free(key);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	action_name = crm_strdup_printf("%s%s '%s' %s",
	prefix, action->task, op, action_host);

	} else if (action->rsc && action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->uuid, action_host);

	} else if (action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->task, action_host);

	} else {
	action_name = crm_strdup_printf("%s", action->uuid);
	}

	if (verbose) {
	char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);

	free(action_name);
	action_name = with_id;
	}
	return action_name;
	}

	/*!
	* \internal
	* \brief Display the status of a cluster
	*
	* \param[in,out] data_set Cluster working set
	* \param[in] show_opts How to modify display (as pcmk_show_opt_e flags)
	* \param[in] section_opts Sections to display (as pcmk_section_e flags)
	* \param[in] title What to use as list title
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_cluster_status(pe_working_set_t *data_set, uint32_t show_opts,
	uint32_t section_opts, const char *title,
	bool print_spacer)
	{
	pcmk__output_t *out = data_set->priv;
	GList *all = NULL;
	crm_exit_t stonith_rc = 0;
	enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;

	section_opts \|= pcmk_section_nodes \| pcmk_section_resources;
	show_opts \|= pcmk_show_inactive_rscs \| pcmk_show_failed_detail;

	all = g_list_prepend(all, (gpointer) "*");

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "%s", title);
	out->message(out, "cluster-status",
	data_set, state, stonith_rc, NULL,
	false, section_opts, show_opts, NULL, all, all);
	out->end_list(out);

	g_list_free(all);
	}

	/*!
	* \internal
	* \brief Display a summary of all actions scheduled in a transition
	*
	* \param[in,out] data_set Cluster working set (fully scheduled)
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_transition_summary(pe_working_set_t *data_set, bool print_spacer)
	{
	pcmk__output_t *out = data_set->priv;

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "Transition Summary");
	pcmk__output_actions(data_set);
	out->end_list(out);
	}

	/*!
	* \internal
	* \brief Reset a cluster working set's input, output, date, and flags
	*
	* \param[in,out] data_set Cluster working set
	* \param[in] input What to set as cluster input
	* \param[in] out What to set as cluster output object
	* \param[in] use_date What to set as cluster's current timestamp
	* \param[in] flags Group of enum pcmk_scheduler_flags to set
	*/
	static void
	reset(pe_working_set_t data_set, xmlNodePtr input, pcmk__output_t out,
	const char *use_date, unsigned int flags)
	{
	data_set->input = input;
	data_set->priv = out;
	set_effective_date(data_set, true, use_date);
	if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
	pe__set_working_set_flags(data_set, pcmk_sched_sanitized);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pe__set_working_set_flags(data_set, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	+ pe__set_working_set_flags(data_set, pcmk_sched_show_utilization);
	}
	}

	/*!
	* \brief Write out a file in dot(1) format describing the actions that will
	* be taken by the scheduler in response to an input CIB file.
	*
	* \param[in,out] data_set Working set for the cluster
	* \param[in] dot_file The filename to write
	* \param[in] all_actions Write all actions, even those that are optional
	* or are on unmanaged resources
	* \param[in] verbose Add extra information, such as action IDs, to the
	* output
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_sim_dotfile(pe_working_set_t data_set, const char dot_file,
	bool all_actions, bool verbose)
	{
	GList *iter = NULL;
	FILE *dot_strm = fopen(dot_file, "w");

	if (dot_strm == NULL) {
	return errno;
	}

	fprintf(dot_strm, " digraph \"g\" {\n");
	for (iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;
	const char *style = "dashed";
	const char *font = "black";
	const char *color = "black";
	char *action_name = create_action_name(action, verbose);

	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	font = "orange";
	}

	if (pcmk_is_set(action->flags, pe_action_dumped)) {
	style = "bold";
	color = "green";

	} else if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
	color = "red";
	font = "purple";
	if (!all_actions) {
	goto do_not_write;
	}

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	color = "blue";
	if (!all_actions) {
	goto do_not_write;
	}

	} else {
	color = "red";
	CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pe_action_runnable));
	}

	pe__set_action_flags(action, pe_action_dumped);
	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
	action_name, style, color, font);
	do_not_write:
	free(action_name);
	}

	for (iter = data_set->actions; iter != NULL; iter = iter->next) {
	pe_action_t action = (pe_action_t ) iter->data;

	for (GList *before_iter = action->actions_before;
	before_iter != NULL; before_iter = before_iter->next) {

	pe_action_wrapper_t *before = before_iter->data;

	char *before_name = NULL;
	char *after_name = NULL;
	const char *style = "dashed";
	bool optional = true;

	if (before->state == pe_link_dumped) {
	optional = false;
	style = "bold";
	} else if (before->type == pe_order_none) {
	continue;
	} else if (pcmk_is_set(before->action->flags, pe_action_dumped)
	&& pcmk_is_set(action->flags, pe_action_dumped)
	&& before->type != pe_order_load) {
	optional = false;
	}

	if (all_actions \|\| !optional) {
	before_name = create_action_name(before->action, verbose);
	after_name = create_action_name(action, verbose);
	fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
	before_name, after_name, style);
	free(before_name);
	free(after_name);
	}
	}
	}

	fprintf(dot_strm, "}\n");
	fflush(dot_strm);
	fclose(dot_strm);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Profile the configuration updates and scheduler actions in a single
	* CIB file, printing the profiling timings.
	*
	* \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in] xml_file The CIB file to profile
	* \param[in] repeat Number of times to run
	* \param[in,out] data_set Working set for the cluster
	* \param[in] use_date The date to set the cluster's time to (may be NULL)
	*/
	static void
	profile_file(const char xml_file, long long repeat, pe_working_set_t data_set,
	const char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	xmlNode *cib_object = NULL;
	clock_t start = 0;
	clock_t end;
	unsigned long long data_set_flags = pcmk_sched_no_compat;

	CRM_ASSERT(out != NULL);

	cib_object = filename2xml(xml_file);
	start = clock();

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return;
	}

	if (pcmk_is_set(data_set->flags, pcmk_sched_output_scores)) {
	data_set_flags \|= pcmk_sched_output_scores;
	}
	- if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	- data_set_flags \|= pe_flag_show_utilization;
	+ if (pcmk_is_set(data_set->flags, pcmk_sched_show_utilization)) {
	+ data_set_flags \|= pcmk_sched_show_utilization;
	}

	for (int i = 0; i < repeat; ++i) {
	xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);

	data_set->input = input;
	set_effective_date(data_set, false, use_date);
	pcmk__schedule_actions(input, data_set_flags, data_set);
	pe_reset_working_set(data_set);
	}

	end = clock();
	out->message(out, "profile", xml_file, start, end);
	}

	void
	pcmk__profile_dir(const char dir, long long repeat, pe_working_set_t data_set,
	const char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	struct dirent **namelist;

	int file_num = scandir(dir, &namelist, 0, alphasort);

	CRM_ASSERT(out != NULL);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX];

	out->begin_list(out, NULL, NULL, "Timings");

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
	".xml")) {
	free(namelist[file_num]);
	continue;
	}
	snprintf(buffer, sizeof(buffer), "%s/%s",
	dir, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	profile_file(buffer, repeat, data_set, use_date);
	}
	free(namelist[file_num]);
	}
	free(namelist);

	out->end_list(out);
	}
	}

	/*!
	* \brief Set the date of the cluster, either to the value given by
	* \p use_date, or to the "execution-date" value in the CIB.
	*
	* \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in,out] data_set Working set for the cluster
	* \param[in] print_original If \p true, the "execution-date" should
	* also be printed
	* \param[in] use_date The date to set the cluster's time to
	* (may be NULL)
	*/
	static void
	set_effective_date(pe_working_set_t *data_set, bool print_original,
	const char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	time_t original_date = 0;

	CRM_ASSERT(out != NULL);

	crm_element_value_epoch(data_set->input, "execution-date", &original_date);

	if (use_date) {
	data_set->now = crm_time_new(use_date);
	out->info(out, "Setting effective cluster time: %s", use_date);
	crm_time_log(LOG_NOTICE, "Pretending 'now' is", data_set->now,
	crm_time_log_date \| crm_time_log_timeofday);

	} else if (original_date != 0) {
	data_set->now = pcmk__copy_timet(original_date);

	if (print_original) {
	char *when = crm_time_as_string(data_set->now,
	crm_time_log_date\|crm_time_log_timeofday);

	out->info(out, "Using the original execution date of: %s", when);
	free(when);
	}
	}
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a pseudo-action in a graph
	*
	* \param[in,out] graph Graph to update with pseudo-action result
	* \param[in,out] action Pseudo-action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-pseudo-action", node, task);

	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate executing a resource action in a graph
	*
	* \param[in,out] graph Graph to update with resource action result
	* \param[in,out] action Resource action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_resource_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	int rc;
	lrmd_event_data_t *op = NULL;
	int target_outcome = PCMK_OCF_OK;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *resource = NULL;
	const char *rprovider = NULL;
	const char *resource_config_name = NULL;
	const char *operation = crm_element_value(action->xml, "operation");
	const char *target_rc_s = crm_meta_value(action->params,
	XML_ATTR_TE_TARGET_RC);

	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	char *uuid = NULL;
	const char *router_node = crm_element_value(action->xml,
	XML_LRM_ATTR_ROUTER_NODE);

	// Certain actions don't need to be displayed or history entries
	if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	crm_debug("No history injection for %s op on %s", operation, node);
	goto done; // Confirm action and update graph
	}

	if (action_rsc == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "Bad");
	free(node);
	return EPROTO;
	}

	/* A resource might be known by different names in the configuration and in
	* the action (for example, a clone instance). Grab the configuration name
	* (which is preferred when writing history), and if necessary, the instance
	* name.
	*/
	resource_config_name = crm_element_value(action_rsc, XML_ATTR_ID);
	if (resource_config_name == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "No ID");
	free(node);
	return EPROTO;
	}
	resource = resource_config_name;
	if (pe_find_resource(fake_resource_list, resource) == NULL) {
	const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);

	if ((longname != NULL)
	&& (pe_find_resource(fake_resource_list, longname) != NULL)) {
	resource = longname;
	}
	}

	// Certain actions need to be displayed but don't need history entries
	if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE,
	PCMK_ACTION_META_DATA, NULL)) {
	out->message(out, "inject-rsc-action", resource, operation, node,
	(guint) 0);
	goto done; // Confirm action and update graph
	}

	rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
	rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);

	pcmk__scan_min_int(target_rc_s, &target_outcome, 0);

	CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
	cib_sync_call\|cib_scope_local) == pcmk_ok);

	// Ensure the action node is in the CIB
	uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	cib_node = pcmk__inject_node(fake_cib, node,
	((router_node == NULL)? uuid: node));
	free(uuid);
	CRM_ASSERT(cib_node != NULL);

	// Add a history entry for the action
	cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
	resource_config_name,
	rclass, rtype, rprovider);
	if (cib_resource == NULL) {
	crm_err("Could not simulate action %d history for resource %s",
	action->id, resource);
	free(node);
	free_xml(cib_node);
	return EINVAL;
	}

	// Simulate and display an executor event for the action result
	op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	target_outcome, "User-injected result");
	out->message(out, "inject-rsc-action", resource, op->op_type, node,
	op->interval_ms);

	// Check whether action is in a list of desired simulated failures
	for (const GList *iter = fake_op_fail_list;
	iter != NULL; iter = iter->next) {
	const char spec = (const char ) iter->data;
	char *key = NULL;
	const char *match_name = NULL;

	// Allow user to specify anonymous clone with or without instance number
	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource;
	}
	free(key);

	// If not found, try the resource's name in the configuration
	if ((match_name == NULL)
	&& (strcmp(resource, resource_config_name) != 0)) {

	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
	op->op_type, op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource_config_name;
	}
	free(key);
	}

	if (match_name == NULL) {
	continue; // This failed action entry doesn't match
	}

	// ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	if (rc != 1) {
	out->err(out, "Invalid failed operation '%s' "
	"(result code must be integer)", spec);
	continue; // Keep checking other list entries
	}

	out->info(out, "Pretending action %d failed with rc=%d",
	action->id, op->rc);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	graph->abort_priority = INFINITY;
	pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
	op->interval_ms, op->rc);
	break;
	}

	pcmk__inject_action_result(cib_resource, op, target_outcome);
	lrmd_free_event(op);
	rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(node);
	free_xml(cib_node);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a cluster action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Cluster action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-cluster-action", node, task, rsc);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a fencing action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Fencing action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_fencing_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *op = crm_meta_value(action->params, "stonith_action");
	char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);

	out->message(out, "inject-fencing-action", target, op);

	if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	int rc = pcmk_ok;
	GString *xpath = g_string_sized_new(512);

	// Set node state to offline
	xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
	false);

	CRM_ASSERT(cib_node != NULL);
	crm_xml_add(cib_node, XML_ATTR_ORIGIN, __func__);
	rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	// Simulate controller clearing node's resource history and attributes
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']/" XML_CIB_TAG_LRM,
	NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	g_string_truncate(xpath, 0);
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']"
	"/" XML_TAG_TRANSIENT_NODEATTRS, NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	free_xml(cib_node);
	g_string_free(xpath, TRUE);
	}

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	free(target);
	return pcmk_rc_ok;
	}

	enum pcmk__graph_status
	pcmk__simulate_transition(pe_working_set_t data_set, cib_t cib,
	const GList *op_fail_list)
	{
	pcmk__graph_t *transition = NULL;
	enum pcmk__graph_status graph_rc;

	pcmk__graph_functions_t simulation_fns = {
	simulate_pseudo_action,
	simulate_resource_action,
	simulate_cluster_action,
	simulate_fencing_action,
	};

	out = data_set->priv;

	fake_cib = cib;
	fake_op_fail_list = op_fail_list;

	if (!out->is_quiet(out)) {
	out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	}

	pcmk__set_graph_functions(&simulation_fns);
	transition = pcmk__unpack_graph(data_set->graph, crm_system_name);
	pcmk__log_graph(LOG_DEBUG, transition);

	fake_resource_list = data_set->resources;
	do {
	graph_rc = pcmk__execute_graph(transition);
	} while (graph_rc == pcmk__graph_active);
	fake_resource_list = NULL;

	if (graph_rc != pcmk__graph_complete) {
	out->err(out, "Transition failed: %s",
	pcmk__graph_status2text(graph_rc));
	pcmk__log_graph(LOG_ERR, transition);
	out->err(out, "An invalid transition was produced");
	}
	pcmk__free_graph(transition);

	if (!out->is_quiet(out)) {
	// If not quiet, we'll need the resulting CIB for later display
	xmlNode *cib_object = NULL;
	int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
	cib_sync_call\|cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	pe_reset_working_set(data_set);
	data_set->input = cib_object;
	out->end_list(out);
	}
	return graph_rc;
	}

	int
	pcmk__simulate(pe_working_set_t data_set, pcmk__output_t out,
	const pcmk_injections_t *injections, unsigned int flags,
	uint32_t section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	int printed = pcmk_rc_no_output;
	int rc = pcmk_rc_ok;
	xmlNodePtr input = NULL;
	cib_t *cib = NULL;

	rc = cib__signon_query(out, &cib, &input);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}

	reset(data_set, input, out, use_date, flags);
	cluster_status(data_set);

	if ((cib->variant == cib_native)
	&& pcmk_is_set(section_opts, pcmk_section_times)) {
	if (pcmk__our_nodename == NULL) {
	// Currently used only in the times section
	pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
	}
	data_set->localhost = pcmk__our_nodename;
	}

	if (!out->is_quiet(out)) {
	const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending);

	if (pcmk_is_set(data_set->flags, pcmk_sched_in_maintenance)) {
	printed = out->message(out, "maint-mode", data_set->flags);
	}

	if (data_set->disabled_resources \|\| data_set->blocked_resources) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	printed = out->info(out,
	"%d of %d resource instances DISABLED and "
	"%d BLOCKED from further action due to failure",
	data_set->disabled_resources,
	data_set->ninstances,
	data_set->blocked_resources);
	}

	/* Most formatted output headers use caps for each word, but this one
	* only has the first word capitalized for compatibility with pcs.
	*/
	print_cluster_status(data_set, (show_pending? pcmk_show_pending : 0),
	section_opts, "Current cluster status",
	(printed == pcmk_rc_ok));
	printed = pcmk_rc_ok;
	}

	// If the user requested any injections, handle them
	if ((injections->node_down != NULL)
	\|\| (injections->node_fail != NULL)
	\|\| (injections->node_up != NULL)
	\|\| (injections->op_inject != NULL)
	\|\| (injections->ticket_activate != NULL)
	\|\| (injections->ticket_grant != NULL)
	\|\| (injections->ticket_revoke != NULL)
	\|\| (injections->ticket_standby != NULL)
	\|\| (injections->watchdog != NULL)) {

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	pcmk__inject_scheduler_input(data_set, cib, injections);
	printed = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
	if (rc != pcmk_rc_ok) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}

	cleanup_calculations(data_set);
	reset(data_set, input, out, use_date, flags);
	cluster_status(data_set);
	}

	if (input_file != NULL) {
	rc = write_xml_file(input, input_file, FALSE);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}
	}

	if (pcmk_any_flags_set(flags, pcmk_sim_process \| pcmk_sim_simulate)) {
	pcmk__output_t *logger_out = NULL;
	unsigned long long data_set_flags = pcmk_sched_no_compat;

	if (pcmk_is_set(data_set->flags, pcmk_sched_output_scores)) {
	data_set_flags \|= pcmk_sched_output_scores;
	}
	- if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	- data_set_flags \|= pe_flag_show_utilization;
	+ if (pcmk_is_set(data_set->flags, pcmk_sched_show_utilization)) {
	+ data_set_flags \|= pcmk_sched_show_utilization;
	}

	if (pcmk_all_flags_set(data_set->flags,
	pcmk_sched_output_scores
	- \|pe_flag_show_utilization)) {
	+ \|pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL,
	"Assignment Scores and Utilization Information");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(data_set->flags, pcmk_sched_output_scores)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Assignment Scores");
	printed = pcmk_rc_ok;

	- } else if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	+ } else if (pcmk_is_set(data_set->flags, pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Utilization Information");
	printed = pcmk_rc_ok;

	} else {
	rc = pcmk__log_output_new(&logger_out);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}
	pe__register_messages(logger_out);
	pcmk__register_lib_messages(logger_out);
	data_set->priv = logger_out;
	}

	pcmk__schedule_actions(input, data_set_flags, data_set);

	if (logger_out == NULL) {
	out->end_list(out);
	} else {
	logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger_out);
	data_set->priv = out;
	}

	input = NULL; /* Don't try and free it twice */

	if (graph_file != NULL) {
	rc = write_xml_file(data_set->graph, graph_file, FALSE);
	if (rc < 0) {
	rc = pcmk_rc_graph_error;
	goto simulate_done;
	}
	}

	if (dot_file != NULL) {
	rc = write_sim_dotfile(data_set, dot_file,
	pcmk_is_set(flags, pcmk_sim_all_actions),
	pcmk_is_set(flags, pcmk_sim_verbose));
	if (rc != pcmk_rc_ok) {
	rc = pcmk_rc_dot_error;
	goto simulate_done;
	}
	}

	if (!out->is_quiet(out)) {
	print_transition_summary(data_set, printed == pcmk_rc_ok);
	}
	}

	rc = pcmk_rc_ok;

	if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
	goto simulate_done;
	}

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	if (pcmk__simulate_transition(data_set, cib, injections->op_fail)
	!= pcmk__graph_complete) {
	rc = pcmk_rc_invalid_transition;
	}

	if (out->is_quiet(out)) {
	goto simulate_done;
	}

	set_effective_date(data_set, true, use_date);

	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pe__set_working_set_flags(data_set, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	+ pe__set_working_set_flags(data_set, pcmk_sched_show_utilization);
	}

	cluster_status(data_set);
	print_cluster_status(data_set, 0, section_opts, "Revised Cluster Status",
	true);

	simulate_done:
	cib__clean_up_connection(&cib);
	return rc;
	}

	int
	pcmk_simulate(xmlNodePtr xml, pe_working_set_t data_set,
	const pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__simulate(data_set, out, injections, flags, section_opts,
	use_date, input_file, graph_file, dot_file);
	pcmk__xml_output_finish(out, xml);
	return rc;
	}
	diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
	index d7369a9b4f..c7c14d6892 100644
	--- a/tools/crm_simulate.c
	+++ b/tools/crm_simulate.c
	@@ -1,588 +1,588 @@
	/*
	* Copyright 2009-2023 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <time.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#define SUMMARY "crm_simulate - simulate a Pacemaker cluster's response to events"

	struct {
	char *dot_file;
	char *graph_file;
	gchar *input_file;
	pcmk_injections_t *injections;
	unsigned int flags;
	gchar *output_file;
	long long repeat;
	gboolean store;
	gchar *test_dir;
	char *use_date;
	char *xml_file;
	} options = {
	.flags = pcmk_sim_show_pending \| pcmk_sim_sanitized,
	.repeat = 1
	};

	uint32_t section_opts = 0;
	char *temp_shadow = NULL;
	crm_exit_t exit_code = CRM_EX_OK;

	#define INDENT " "

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static gboolean
	all_actions_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_all_actions;
	return TRUE;
	}

	static gboolean
	attrs_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	section_opts \|= pcmk_section_attributes;
	return TRUE;
	}

	static gboolean
	failcounts_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	section_opts \|= pcmk_section_failcounts \| pcmk_section_failures;
	return TRUE;
	}

	static gboolean
	in_place_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.store = TRUE;
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	return TRUE;
	}

	static gboolean
	live_check_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (options.xml_file) {
	free(options.xml_file);
	}

	options.xml_file = NULL;
	options.flags &= ~pcmk_sim_sanitized;
	return TRUE;
	}

	static gboolean
	node_down_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->node_down = g_list_append(options.injections->node_down, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	node_fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->node_fail = g_list_append(options.injections->node_fail, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	node_up_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__simulate_node_config = true;
	options.injections->node_up = g_list_append(options.injections->node_up, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	op_fail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	options.injections->op_fail = g_list_append(options.injections->op_fail, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	op_inject_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->op_inject = g_list_append(options.injections->op_inject, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	pending_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_show_pending;
	return TRUE;
	}

	static gboolean
	process_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	return TRUE;
	}

	static gboolean
	quorum_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.injections->quorum, optarg);
	return TRUE;
	}

	static gboolean
	save_dotfile_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	pcmk__str_update(&options.dot_file, optarg);
	return TRUE;
	}

	static gboolean
	save_graph_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process;
	pcmk__str_update(&options.graph_file, optarg);
	return TRUE;
	}

	static gboolean
	show_scores_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_show_scores;
	return TRUE;
	}

	static gboolean
	simulate_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_simulate;
	return TRUE;
	}

	static gboolean
	ticket_activate_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_activate = g_list_append(options.injections->ticket_activate, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_grant_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_grant = g_list_append(options.injections->ticket_grant, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_revoke_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_revoke = g_list_append(options.injections->ticket_revoke, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	ticket_standby_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.injections->ticket_standby = g_list_append(options.injections->ticket_standby, g_strdup(optarg));
	return TRUE;
	}

	static gboolean
	utilization_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.flags \|= pcmk_sim_process \| pcmk_sim_show_utilization;
	return TRUE;
	}

	static gboolean
	watchdog_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.injections->watchdog, optarg);
	return TRUE;
	}

	static gboolean
	xml_file_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.xml_file, optarg);
	options.flags \|= pcmk_sim_sanitized;
	return TRUE;
	}

	static gboolean
	xml_pipe_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	pcmk__str_update(&options.xml_file, "-");
	options.flags \|= pcmk_sim_sanitized;
	return TRUE;
	}

	static GOptionEntry operation_entries[] = {
	{ "run", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, process_cb,
	"Process the supplied input and show what actions the cluster will take in response",
	NULL },
	{ "simulate", 'S', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, simulate_cb,
	"Like --run, but also simulate taking those actions and show the resulting new status",
	NULL },
	{ "in-place", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, in_place_cb,
	"Like --simulate, but also store the results back to the input file",
	NULL },
	{ "show-attrs", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, attrs_cb,
	"Show node attributes",
	NULL },
	{ "show-failcounts", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, failcounts_cb,
	"Show resource fail counts",
	NULL },
	{ "show-scores", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_scores_cb,
	"Show allocation scores",
	NULL },
	{ "show-utilization", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, utilization_cb,
	"Show utilization information",
	NULL },
	{ "profile", 'P', 0, G_OPTION_ARG_FILENAME, &options.test_dir,
	"Process all the XML files in the named directory to create profiling data",
	"DIR" },
	{ "repeat", 'N', 0, G_OPTION_ARG_INT, &options.repeat,
	"With --profile, repeat each test N times and print timings",
	"N" },
	/* Deprecated */
	{ "pending", 'j', G_OPTION_FLAG_NO_ARG\|G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, pending_cb,
	"Display pending state if 'record-pending' is enabled",
	NULL },

	{ NULL }
	};

	static GOptionEntry synthetic_entries[] = {
	{ "node-up", 'u', 0, G_OPTION_ARG_CALLBACK, node_up_cb,
	"Simulate bringing a node online",
	"NODE" },
	{ "node-down", 'd', 0, G_OPTION_ARG_CALLBACK, node_down_cb,
	"Simulate taking a node offline",
	"NODE" },
	{ "node-fail", 'f', 0, G_OPTION_ARG_CALLBACK, node_fail_cb,
	"Simulate a node failing",
	"NODE" },
	{ "op-inject", 'i', 0, G_OPTION_ARG_CALLBACK, op_inject_cb,
	"Generate a failure for the cluster to react to in the simulation.\n"
	INDENT "See `Operation Specification` help for more information.",
	"OPSPEC" },
	{ "op-fail", 'F', 0, G_OPTION_ARG_CALLBACK, op_fail_cb,
	"If the specified task occurs during the simulation, have it fail with return code ${rc}.\n"
	INDENT "The transition will normally stop at the failed action.\n"
	INDENT "Save the result with --save-output and re-run with --xml-file.\n"
	INDENT "See `Operation Specification` help for more information.",
	"OPSPEC" },
	{ "set-datetime", 't', 0, G_OPTION_ARG_STRING, &options.use_date,
	"Set date/time (ISO 8601 format, see https://en.wikipedia.org/wiki/ISO_8601)",
	"DATETIME" },
	{ "quorum", 'q', 0, G_OPTION_ARG_CALLBACK, quorum_cb,
	"Set to '1' (or 'true') to indicate cluster has quorum",
	"QUORUM" },
	{ "watchdog", 'w', 0, G_OPTION_ARG_CALLBACK, watchdog_cb,
	"Set to '1' (or 'true') to indicate cluster has an active watchdog device",
	"DEVICE" },
	{ "ticket-grant", 'g', 0, G_OPTION_ARG_CALLBACK, ticket_grant_cb,
	"Simulate granting a ticket",
	"TICKET" },
	{ "ticket-revoke", 'r', 0, G_OPTION_ARG_CALLBACK, ticket_revoke_cb,
	"Simulate revoking a ticket",
	"TICKET" },
	{ "ticket-standby", 'b', 0, G_OPTION_ARG_CALLBACK, ticket_standby_cb,
	"Simulate making a ticket standby",
	"TICKET" },
	{ "ticket-activate", 'e', 0, G_OPTION_ARG_CALLBACK, ticket_activate_cb,
	"Simulate activating a ticket",
	"TICKET" },

	{ NULL }
	};

	static GOptionEntry artifact_entries[] = {
	{ "save-input", 'I', 0, G_OPTION_ARG_FILENAME, &options.input_file,
	"Save the input configuration to the named file",
	"FILE" },
	{ "save-output", 'O', 0, G_OPTION_ARG_FILENAME, &options.output_file,
	"Save the output configuration to the named file",
	"FILE" },
	{ "save-graph", 'G', 0, G_OPTION_ARG_CALLBACK, save_graph_cb,
	"Save the transition graph (XML format) to the named file",
	"FILE" },
	{ "save-dotfile", 'D', 0, G_OPTION_ARG_CALLBACK, save_dotfile_cb,
	"Save the transition graph (DOT format) to the named file",
	"FILE" },
	{ "all-actions", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, all_actions_cb,
	"Display all possible actions in DOT graph (even if not part of transition)",
	NULL },

	{ NULL }
	};

	static GOptionEntry source_entries[] = {
	{ "live-check", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, live_check_cb,
	"Connect to CIB manager and use the current CIB contents as input",
	NULL },
	{ "xml-file", 'x', 0, G_OPTION_ARG_CALLBACK, xml_file_cb,
	"Retrieve XML from the named file",
	"FILE" },
	{ "xml-pipe", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, xml_pipe_cb,
	"Retrieve XML from stdin",
	NULL },

	{ NULL }
	};

	static int
	setup_input(pcmk__output_t out, const char input, const char *output,
	GError **error)
	{
	int rc = pcmk_rc_ok;
	xmlNode *cib_object = NULL;
	char *local_output = NULL;

	if (input == NULL) {
	/* Use live CIB */
	rc = cib__signon_query(out, NULL, &cib_object);
	if (rc != pcmk_rc_ok) {
	// cib__signon_query() outputs any relevant error
	return rc;
	}

	} else if (pcmk__str_eq(input, "-", pcmk__str_casei)) {
	cib_object = filename2xml(NULL);

	} else {
	cib_object = filename2xml(input);
	}

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return pcmk_rc_transform_failed;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return pcmk_rc_schema_validation;
	}

	if (output == NULL) {
	char *pid = pcmk__getpid_s();

	local_output = get_shadow_file(pid);
	temp_shadow = strdup(local_output);
	output = local_output;
	free(pid);
	}

	rc = write_xml_file(cib_object, output, FALSE);
	free_xml(cib_object);
	cib_object = NULL;

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT,
	"Could not create '%s': %s", output, pcmk_rc_str(rc));
	return rc;
	} else {
	setenv("CIB_file", output, 1);
	free(local_output);
	return pcmk_rc_ok;
	}
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Display only essential output",
	NULL },

	{ NULL }
	};

	const char *description = "Operation Specification:\n\n"
	"The OPSPEC in any command line option is of the form\n"
	"${resource}_${task}_${interval_in_ms}@${node}=${rc}\n"
	"(memcached_monitor_20000@bart.example.com=7, for example).\n"
	"${rc} is an OCF return code. For more information on these\n"
	"return codes, refer to https://clusterlabs.org/pacemaker/doc/2.1/Pacemaker_Administration/html/agents.html#ocf-return-codes\n\n"
	"Examples:\n\n"
	"Pretend a recurring monitor action found memcached stopped on node\n"
	"fred.example.com and, during recovery, that the memcached stop\n"
	"action failed:\n\n"
	"\tcrm_simulate -LS --op-inject memcached:0_monitor_20000@bart.example.com=7 "
	"--op-fail memcached:0_stop_0@fred.example.com=1 --save-output /tmp/memcached-test.xml\n\n"
	"Now see what the reaction to the stop failed would be:\n\n"
	"\tcrm_simulate -S --xml-file /tmp/memcached-test.xml\n\n";

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	pcmk__add_main_args(context, extra_prog_entries);
	g_option_context_set_description(context, description);

	pcmk__add_arg_group(context, "operations", "Operations:",
	"Show operations options", operation_entries);
	pcmk__add_arg_group(context, "synthetic", "Synthetic Cluster Events:",
	"Show synthetic cluster event options", synthetic_entries);
	pcmk__add_arg_group(context, "artifact", "Artifact Options:",
	"Show artifact options", artifact_entries);
	pcmk__add_arg_group(context, "source", "Data Source:",
	"Show data source options", source_entries);

	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	pe_working_set_t *data_set = NULL;
	pcmk__output_t *out = NULL;

	GError *error = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINOP");
	GOptionContext *context = build_arg_context(args, &output_group);

	options.injections = calloc(1, sizeof(pcmk_injections_t));
	if (options.injections == NULL) {
	rc = ENOMEM;
	goto done;
	}

	/* This must come before g_option_context_parse_strv. */
	options.xml_file = strdup("-");

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_simulate", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	fprintf(stderr, "Error creating output format %s: %s\n",
	args->output_ty, pcmk_rc_str(rc));
	exit_code = CRM_EX_ERROR;
	goto done;
	}

	if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches) &&
	!pcmk_is_set(options.flags, pcmk_sim_show_scores) &&
	!pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
	pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
	} else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	out->quiet = args->quiet;

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (args->verbosity > 0) {
	options.flags \|= pcmk_sim_verbose;

	#ifdef PCMK__COMPAT_2_0
	/* Redirect stderr to stdout so we can grep the output */
	close(STDERR_FILENO);
	dup2(STDOUT_FILENO, STDERR_FILENO);
	#endif
	}

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	rc = ENOMEM;
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not allocate working set");
	goto done;
	}

	if (pcmk_is_set(options.flags, pcmk_sim_show_scores)) {
	pe__set_working_set_flags(data_set, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(options.flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	+ pe__set_working_set_flags(data_set, pcmk_sched_show_utilization);
	}
	pe__set_working_set_flags(data_set, pcmk_sched_no_compat);

	if (options.test_dir != NULL) {
	data_set->priv = out;
	pcmk__profile_dir(options.test_dir, options.repeat, data_set, options.use_date);
	rc = pcmk_rc_ok;
	goto done;
	}

	rc = setup_input(out, options.xml_file,
	options.store? options.xml_file : options.output_file,
	&error);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk__simulate(data_set, out, options.injections, options.flags, section_opts,
	options.use_date, options.input_file, options.graph_file,
	options.dot_file);

	done:
	pcmk__output_and_clear_error(&error, NULL);

	/* There sure is a lot to free in options. */
	free(options.dot_file);
	free(options.graph_file);
	g_free(options.input_file);
	g_free(options.output_file);
	g_free(options.test_dir);
	free(options.use_date);
	free(options.xml_file);

	pcmk_free_injections(options.injections);
	pcmk__free_arg_context(context);
	g_strfreev(processed_args);

	if (data_set) {
	pe_free_working_set(data_set);
	}

	fflush(stderr);

	if (temp_shadow) {
	unlink(temp_shadow);
	free(temp_shadow);
	}

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	}

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}

	pcmk__unregister_formats();
	crm_exit(exit_code);
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Jun 25, 2:22 AM (14 h, 10 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1951873
Default Alt Text: (130 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions