No OneTemporary
Actions

Size

163 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/include/pcmki/pcmki_sched_utils.h b/include/pcmki/pcmki_sched_utils.h
	index fa1e0d6ff9..c53e9f95bb 100644
	--- a/include/pcmki/pcmki_sched_utils.h
	+++ b/include/pcmki/pcmki_sched_utils.h
	@@ -1,65 +1,63 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PENGINE_AUTILS__H
	# define PENGINE_AUTILS__H

	#include <stdbool.h> // bool
	#include <glib.h> // GList, GHashTable, gboolean, guint
	#include <crm/lrmd.h> // lrmd_event_data_t
	#include <crm/cib.h> // cib_t
	#include <crm/pengine/pe_types.h>
	#include <crm/pengine/internal.h>
	#include <pcmki/pcmki_scheduler.h>
	#include <pcmki/pcmki_transition.h>
	#include <pacemaker.h>

	/* Constraint helper functions */
	pcmk__colocation_t invert_constraint(pcmk__colocation_t constraint);

	pe__location_t copy_constraint(pe__location_t constraint);

	GHashTable pcmk__copy_node_table(GHashTable nodes);
	GList pcmk__copy_node_list(const GList list, bool reset);
	GList pcmk__sort_nodes(GList nodes, pe_node_t *active_node,
	pe_working_set_t *data_set);

	bool pcmk__node_available(const pe_node_t *node);
	bool pcmk__any_node_available(GHashTable *nodes);

	pe_resource_t find_compatible_child(pe_resource_t local_child,
	pe_resource_t *rsc, enum rsc_role_e filter,
	gboolean current,
	pe_working_set_t *data_set);
	pe_resource_t find_compatible_child_by_node(pe_resource_t local_child, pe_node_t * local_node, pe_resource_t * rsc,
	enum rsc_role_e filter, gboolean current);
	gboolean is_child_compatible(pe_resource_t child_rsc, pe_node_t local_node, enum rsc_role_e filter, gboolean current);
	enum pe_action_flags summary_action_flags(pe_action_t * action, GList children, pe_node_t node);
	enum action_tasks clone_child_action(pe_action_t * action);
	int copies_per_node(pe_resource_t * rsc);

	extern int compare_capacity(const pe_node_t * node1, const pe_node_t * node2);
	extern void calculate_utilization(GHashTable * current_utilization,
	GHashTable * utilization, gboolean plus);

	extern void process_utilization(pe_resource_t * rsc, pe_node_t ** prefer, pe_working_set_t * data_set);

	xmlNode pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t *event,
	const char *caller_version, int target_rc,
	const char node, const char origin,
	int level);

	# define LOAD_STOPPED "load_stopped"

	void modify_configuration(pe_working_set_t data_set, cib_t cib,
	pcmk_injections_t *injections);

	-enum transition_status run_simulation(pe_working_set_t * data_set, cib_t cib, GList op_fail_list);
	-
	#endif
	diff --git a/include/pcmki/pcmki_simulate.h b/include/pcmki/pcmki_simulate.h
	index 79859377ad..dabf5ca483 100644
	--- a/include/pcmki/pcmki_simulate.h
	+++ b/include/pcmki/pcmki_simulate.h
	@@ -1,117 +1,86 @@
	/*
	* Copyright 2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMKI_SIMULATE__H
	# define PCMKI_SIMULATE__H

	#include <crm/common/output_internal.h>
	#include <crm/pengine/pe_types.h>
	#include <pacemaker.h>
	#include <stdbool.h>

	/**
	- * \brief Write out a file in dot(1) format describing the actions that will
	- * be taken by the scheduler in response to an input CIB file.
	- *
	- * \param[in] data_set Working set for the cluster.
	- * \param[in] dot_file The filename to write.
	- * \param[in] all_actions Write all actions, even those that are optional or
	- * are on unmanaged resources.
	- * \param[in] verbose Add extra information, such as action IDs, to the
	- * output.
	- *
	- * \return Standard Pacemaker return code
	- */
	-int pcmk__write_sim_dotfile(pe_working_set_t data_set, const char dot_file,
	- bool all_actions, bool verbose);
	-
	-/**
	- * \brief Profile the configuration updates and scheduler actions in a single
	- * CIB file, printing the profiling timings.
	- *
	- * \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	- * object before this function is called.
	- *
	- * \param[in] xml_file The CIB file to profile.
	- * \param[in] repeat Number of times to run.
	- * \param[in] data_set Working set for the cluster.
	- * \param[in] use_date The date to set the cluster's time to (may
	- * be NULL).
	- */
	-void pcmk__profile_file(const char xml_file, long long repeat, pe_working_set_t data_set,
	- char *use_date);
	-
	-/**
	+ * \internal
	* \brief Profile the configuration updates and scheduler actions in every
	* CIB file in a given directory, printing the profiling timings for
	* each.
	*
	* \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in] dir A directory full of CIB files to be profiled.
	* \param[in] repeat Number of times to run on each input file.
	* \param[in] data_set Working set for the cluster.
	* \param[in] use_date The date to set the cluster's time to (may
	* be NULL).
	*/
	void pcmk__profile_dir(const char dir, long long repeat, pe_working_set_t data_set,
	char *use_date);

	/**
	- * \brief Set the date of the cluster, either to the value given by
	- * \p use_date, or to the "execution-date" value in the CIB.
	+ * \internal
	+ * \brief Simulate executing a transition
	*
	- * \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	- * object before this function is called.
	+ * \param[in] data_set Cluster working set
	+ * \param[in] cib CIB object for scheduler input
	+ * \param[in] op_fail_list List of actions to simulate as failing
	*
	- * \param[in,out] data_set Working set for the cluster.
	- * \param[in] print_original If \p true, the "execution-date" should
	- * also be printed.
	- * \param[in] use_date The date to set the cluster's time to
	- * (may be NULL).
	+ * \return Transition status after simulated execution
	*/
	-void pcmk__set_effective_date(pe_working_set_t data_set, bool print_original, char use_date);
	+enum transition_status pcmk__simulate_transition(pe_working_set_t *data_set,
	+ cib_t *cib,
	+ GList *op_fail_list);

	/**
	+ * \internal
	* \brief Simulate a cluster's response to events.
	*
	* This high-level function essentially implements crm_simulate(8). It operates
	* on an input CIB file and various lists of events that can be simulated. It
	* optionally writes out a variety of artifacts to show the results of the
	* simulation. Output can be modified with various flags.
	*
	* \param[in,out] data_set Working set for the cluster.
	* \param[in,out] out The output functions structure.
	* \param[in] events A structure containing cluster events
	* (node up/down, tickets, injected operations)
	* and related data.
	* \param[in] flags A bitfield of :pcmk_sim_flags to modify
	* operation of the simulation.
	* \param[in] section_opts Which portions of the cluster status output
	* should be displayed?
	* \param[in] use_date The date to set the cluster's time to
	* (may be NULL).
	* \param[in] input_file The source CIB file, which may be overwritten by
	* this function (may be NULL).
	* \param[in] graph_file Where to write the XML-formatted transition graph
	* (may be NULL, in which case no file will be
	* written).
	* \param[in] dot_file Where to write the dot(1) formatted transition
	* graph (may be NULL, in which case no file will
	* be written). See \p pcmk__write_sim_dotfile().
	*
	* \return Standard Pacemaker return code
	*/
	int pcmk__simulate(pe_working_set_t data_set, pcmk__output_t out,
	pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, char use_date, char input_file,
	char graph_file, char dot_file);

	#endif
	diff --git a/lib/pacemaker/libpacemaker_private.h b/lib/pacemaker/libpacemaker_private.h
	index e07561bd64..849640fe17 100644
	--- a/lib/pacemaker/libpacemaker_private.h
	+++ b/lib/pacemaker/libpacemaker_private.h
	@@ -1,273 +1,300 @@
	/*
	* Copyright 2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__LIBPACEMAKER_PRIVATE__H
	# define PCMK__LIBPACEMAKER_PRIVATE__H

	/* This header is for the sole use of libpacemaker, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <crm/pengine/pe_types.h> // pe_action_t, pe_node_t, pe_working_set_t

	// Actions

	G_GNUC_INTERNAL
	void pcmk__update_action_for_orderings(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__log_action(const char pre_text, pe_action_t action, bool details);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_rsc_pseudo_action(pe_resource_t rsc, const char *task,
	bool optional, bool runnable);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_cancel_action(pe_resource_t rsc, const char *name,
	guint interval_ms, pe_node_t *node);

	G_GNUC_INTERNAL
	pe_action_t pcmk__new_shutdown_action(pe_node_t node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__action_locks_rsc_to_node(const pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__deduplicate_action_inputs(pe_action_t *action);

	G_GNUC_INTERNAL
	void pcmk__output_actions(pe_working_set_t *data_set);


	// Producing transition graphs (pcmk_graph_producer.c)

	G_GNUC_INTERNAL
	bool pcmk__graph_has_loop(pe_action_t init_action, pe_action_t action,
	pe_action_wrapper_t *input);

	G_GNUC_INTERNAL
	void pcmk__add_action_to_graph(pe_action_t action, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__create_graph(pe_working_set_t *data_set);


	// Fencing (pcmk_sched_fencing.c)

	G_GNUC_INTERNAL
	void pcmk__order_vs_fence(pe_action_t stonith_op, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__order_vs_unfence(pe_resource_t rsc, pe_node_t node,
	pe_action_t *action, enum pe_ordering order,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__fence_guest(pe_node_t node, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	bool pcmk__node_unfenced(pe_node_t *node);

	G_GNUC_INTERNAL
	bool pcmk__is_unfence_device(const pe_resource_t *rsc,
	const pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	pe_resource_t pcmk__find_constraint_resource(GList rsc_list, const char *id);

	G_GNUC_INTERNAL
	xmlNode pcmk__expand_tags_in_sets(xmlNode xml_obj,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__valid_resource_or_tag(pe_working_set_t data_set, const char id,
	pe_resource_t rsc, pe_tag_t tag);

	G_GNUC_INTERNAL
	bool pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__create_internal_constraints(pe_working_set_t *data_set);


	// Location constraints

	G_GNUC_INTERNAL
	void pcmk__unpack_location(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	pe__location_t pcmk__new_location(const char id, pe_resource_t *rsc,
	int node_weight, const char *discover_mode,
	pe_node_t *foo_node,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_locations(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_location(pe__location_t constraint, pe_resource_t rsc);


	// Colocation constraints

	enum pcmk__coloc_affects {
	pcmk__coloc_affects_nothing = 0,
	pcmk__coloc_affects_location,
	pcmk__coloc_affects_role,
	};

	G_GNUC_INTERNAL
	enum pcmk__coloc_affects pcmk__colocation_affects(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint,
	bool preview);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_weights(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__apply_coloc_to_priority(pe_resource_t *dependent,
	pe_resource_t *primary,
	pcmk__colocation_t *constraint);

	G_GNUC_INTERNAL
	void pcmk__unpack_colocation(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__new_colocation(const char id, const char node_attr, int score,
	pe_resource_t dependent, pe_resource_t primary,
	const char dependent_role, const char primary_role,
	bool influence, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__block_colocated_starts(pe_action_t *action,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__new_ordering(pe_resource_t lh_rsc, char lh_task,
	pe_action_t lh_action, pe_resource_t rh_rsc,
	char rh_task, pe_action_t rh_action,
	enum pe_ordering type, pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__unpack_ordering(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	void pcmk__disable_invalid_orderings(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__order_stops_before_shutdown(pe_node_t *node,
	pe_action_t *shutdown_op,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__apply_orderings(pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Create a new ordering between two resource actions
	*
	* \param[in] lh_rsc Resource for 'first' action
	* \param[in] rh_rsc Resource for 'then' action
	* \param[in] lh_task Action key for 'first' action
	* \param[in] rh_task Action key for 'then' action
	* \param[in] flags Bitmask of enum pe_ordering flags
	* \param[in] data_set Cluster working set to add ordering to
	*/
	#define pcmk__order_resource_actions(lh_rsc, lh_task, rh_rsc, rh_task, \
	flags, data_set) \
	pcmk__new_ordering((lh_rsc), pcmk__op_key((lh_rsc)->id, (lh_task), 0), \
	NULL, \
	(rh_rsc), pcmk__op_key((rh_rsc)->id, (rh_task), 0), \
	NULL, (flags), (data_set))

	#define pcmk__order_starts(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_START, \
	(rsc2), CRMD_ACTION_START, (type), (data_set))

	#define pcmk__order_stops(rsc1, rsc2, type, data_set) \
	pcmk__order_resource_actions((rsc1), CRMD_ACTION_STOP, \
	(rsc2), CRMD_ACTION_STOP, (type), (data_set))

	G_GNUC_INTERNAL
	void pcmk__unpack_rsc_ticket(xmlNode xml_obj, pe_working_set_t data_set);

	G_GNUC_INTERNAL
	bool pcmk__is_failed_remote_node(pe_node_t *node);

	G_GNUC_INTERNAL
	void pcmk__order_remote_connection_actions(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	bool pcmk__rsc_corresponds_to_guest(pe_resource_t rsc, pe_node_t node);

	G_GNUC_INTERNAL
	pe_node_t pcmk__connection_host_for_action(pe_action_t action);

	G_GNUC_INTERNAL
	void pcmk__substitute_remote_addr(pe_resource_t rsc, GHashTable params,
	pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__add_bundle_meta_to_xml(xmlNode args_xml, pe_action_t action);


	// Groups (pcmk_sched_group.c)

	G_GNUC_INTERNAL
	GList pcmk__group_colocated_resources(pe_resource_t rsc,
	pe_resource_t *orig_rsc,
	GList *colocated_rscs);


	// Bundles (pcmk_sched_bundle.c)

	G_GNUC_INTERNAL
	void pcmk__output_bundle_actions(pe_resource_t *rsc);


	+// Injections (pcmk_sched_transition.c)
	+
	+G_GNUC_INTERNAL
	+xmlNode pcmk__inject_node(cib_t cib_conn, const char node, const char uuid);
	+
	+G_GNUC_INTERNAL
	+xmlNode pcmk__inject_node_state_change(cib_t cib_conn, const char *node,
	+ bool up);
	+
	+G_GNUC_INTERNAL
	+xmlNode pcmk__inject_resource_history(pcmk__output_t out, xmlNode *cib_node,
	+ const char *resource,
	+ const char *lrm_name,
	+ const char *rclass,
	+ const char *rtype,
	+ const char *rprovider);
	+
	+G_GNUC_INTERNAL
	+void pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	+ const char resource, const char task,
	+ guint interval_ms, int rc);
	+
	+G_GNUC_INTERNAL
	+xmlNode pcmk__inject_action_result(xmlNode cib_resource,
	+ lrmd_event_data_t *op, int target_rc);
	+
	+
	// Functions applying to more than one variant (pcmk_sched_resource.c)

	G_GNUC_INTERNAL
	GList pcmk__colocated_resources(pe_resource_t rsc, pe_resource_t *orig_rsc,
	GList *colocated_rscs);

	G_GNUC_INTERNAL
	void pcmk__output_resource_actions(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__assign_primitive(pe_resource_t rsc, pe_node_t chosen, bool force);

	G_GNUC_INTERNAL
	bool pcmk__assign_resource(pe_resource_t rsc, pe_node_t node, bool force);

	G_GNUC_INTERNAL
	void pcmk__unassign_resource(pe_resource_t *rsc);

	G_GNUC_INTERNAL
	bool pcmk__threshold_reached(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set,
	pe_resource_t **failed);


	// Functions related to probes (pcmk_sched_probes.c)

	G_GNUC_INTERNAL
	void pcmk__order_probes(pe_working_set_t *data_set);

	G_GNUC_INTERNAL
	void pcmk__schedule_probes(pe_working_set_t *data_set);


	#endif // PCMK__LIBPACEMAKER_PRIVATE__H
	diff --git a/lib/pacemaker/pcmk_sched_transition.c b/lib/pacemaker/pcmk_sched_transition.c
	index 84c9a2783d..6d8f499402 100644
	--- a/lib/pacemaker/pcmk_sched_transition.c
	+++ b/lib/pacemaker/pcmk_sched_transition.c
	@@ -1,850 +1,665 @@
	/*
	* Copyright 2009-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_free_event()
	#include <crm/cib.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	-static pcmk__output_t *out = NULL;
	-static cib_t *fake_cib = NULL;
	-static GList *fake_resource_list = NULL;
	-static GList *fake_op_fail_list = NULL;
	+#include "libpacemaker_private.h"
	+
	gboolean bringing_nodes_online = FALSE;

	#define STATUS_PATH_MAX 512

	#define NEW_NODE_TEMPLATE "//"XML_CIB_TAG_NODE"[@uname='%s']"
	#define NODE_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']"
	#define RSC_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"


	static void
	-inject_transient_attr(xmlNode * cib_node, const char name, const char value)
	+inject_transient_attr(pcmk__output_t out, xmlNode cib_node,
	+ const char name, const char value)
	{
	xmlNode *attrs = NULL;
	xmlNode *instance_attrs = NULL;
	const char *node_uuid = ID(cib_node);

	out->message(out, "inject-attr", name, value, cib_node);

	attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	if (attrs == NULL) {
	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
	}

	instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
	if (instance_attrs == NULL) {
	instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
	crm_xml_add(instance_attrs, XML_ATTR_ID, node_uuid);
	}

	crm_create_nvpair_xml(instance_attrs, NULL, name, value);
	}

	-static void
	-update_failcounts(xmlNode * cib_node, const char resource, const char task,
	- guint interval_ms, int rc)
	+/*!
	+ * \internal
	+ * \brief Inject a fictitious fail count into a scheduler input
	+ *
	+ * \param[in] out Output object for displaying error messages
	+ * \param[in] cib_node Node state XML to inject into
	+ * \param[in] resource ID of resource for fail count to inject
	+ * \param[in] task Action name for fail count to inject
	+ * \param[in] interval_ms Action interval (in milliseconds) for fail count
	+ * \param[in] rc Action result for fail count to inject (if 0, or 7
	+ * when interval_ms is 0, nothing will be injected)
	+ */
	+void
	+pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	+ const char resource, const char task,
	+ guint interval_ms, int rc)
	{
	if (rc == 0) {
	return;

	} else if ((rc == 7) && (interval_ms == 0)) {
	return;

	} else {
	char *name = NULL;
	char *now = pcmk__ttoa(time(NULL));

	name = pcmk__failcount_name(resource, task, interval_ms);
	- inject_transient_attr(cib_node, name, "value++");
	+ inject_transient_attr(out, cib_node, name, "value++");
	free(name);

	name = pcmk__lastfailure_name(resource, task, interval_ms);
	- inject_transient_attr(cib_node, name, now);
	+ inject_transient_attr(out, cib_node, name, now);
	free(name);
	+
	free(now);
	}
	}

	static void
	create_node_entry(cib_t * cib_conn, const char *node)
	{
	int rc = pcmk_ok;
	char *xpath = crm_strdup_printf(NEW_NODE_TEMPLATE, node);

	rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath \| cib_sync_call \| cib_scope_local);

	if (rc == -ENXIO) {
	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);

	crm_xml_add(cib_object, XML_ATTR_ID, node); // Use node name as ID
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
	cib_sync_call \| cib_scope_local);
	/* Not bothering with subsequent query to see if it exists,
	we'll bomb out later in the call to query_node_uuid()... */

	free_xml(cib_object);
	}

	free(xpath);
	}

	static lrmd_event_data_t *
	create_op(xmlNode cib_resource, const char task, guint interval_ms,
	int outcome)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *xop = NULL;

	op = lrmd_new_event(ID(cib_resource), task, interval_ms);
	lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
	op->params = NULL; /* TODO: Fill me in */
	op->t_run = (unsigned int) time(NULL);
	op->t_rcchange = op->t_run;

	op->call_id = 0;
	for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
	xop = pcmk__xe_next(xop)) {

	int tmp = 0;

	crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}
	op->call_id++;

	return op;
	}

	-static xmlNode *
	-inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
	+/*!
	+ * \internal
	+ * \brief Inject a fictitious resource history entry into a scheduler input
	+ *
	+ * \param[in] cib_resource Resource history XML to inject entry into
	+ * \param[in] op Action result to inject
	+ * \param[in] target_rc Expected result for action to inject
	+ *
	+ * \return XML of injected resource history entry
	+ */
	+xmlNode *
	+pcmk__inject_action_result(xmlNode cib_resource, lrmd_event_data_t op,
	+ int target_rc)
	{
	return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
	target_rc, NULL, crm_system_name,
	LOG_TRACE);
	}

	-static xmlNode *
	-inject_node_state(cib_t * cib_conn, const char node, const char uuid)
	+/*!
	+ * \internal
	+ * \brief Inject a fictitious node into a scheduler input
	+ *
	+ * \param[in] cib_conn Scheduler input CIB to inject node into
	+ * \param[in] node Name of node to inject
	+ * \param[in] uuid UUID of node to inject
	+ *
	+ * \return XML of node_state entry for new node
	+ * \note If the global bringing_nodes_online has been set to true, a node entry
	+ * in the configuration section will be added, as well as a node state
	+ * entry in the status section.
	+ */
	+xmlNode *
	+pcmk__inject_node(cib_t cib_conn, const char node, const char *uuid)
	{
	int rc = pcmk_ok;
	xmlNode *cib_object = NULL;
	char *xpath = crm_strdup_printf(NODE_TEMPLATE, node);

	if (bringing_nodes_online) {
	create_node_entry(cib_conn, node);
	}

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	- if (cib_object && ID(cib_object) == NULL) {
	- crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
	+ if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	+ crm_err("Detected multiple node_state entries for xpath=%s, bailing",
	+ xpath);
	crm_log_xml_warn(cib_object, "Duplicates");
	free(xpath);
	crm_exit(CRM_EX_SOFTWARE);
	return NULL; // not reached, but makes static analysis happy
	}

	if (rc == -ENXIO) {
	char *found_uuid = NULL;

	if (uuid == NULL) {
	query_node_uuid(cib_conn, node, &found_uuid, NULL);
	} else {
	found_uuid = strdup(uuid);
	}

	cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
	cib_sync_call \| cib_scope_local);
	free_xml(cib_object);
	free(found_uuid);

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath \| cib_sync_call \| cib_scope_local);
	- crm_trace("injecting node state for %s. rc is %d", node, rc);
	+ crm_trace("Injecting node state for %s (rc=%d)", node, rc);
	}

	free(xpath);
	CRM_ASSERT(rc == pcmk_ok);
	return cib_object;
	}

	-static xmlNode *
	-modify_node(cib_t * cib_conn, char *node, gboolean up)
	+/*!
	+ * \internal
	+ * \brief Inject a fictitious node state change into a scheduler input
	+ *
	+ * \param[in] cib_conn Scheduler input CIB to inject into
	+ * \param[in] node Name of node to inject change for
	+ * \param[in] up If true, change state to online, otherwise offline
	+ *
	+ * \return XML of changed (or added) node state entry
	+ */
	+xmlNode *
	+pcmk__inject_node_state_change(cib_t cib_conn, const char node, bool up)
	{
	- xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);
	+ xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);

	if (up) {
	- crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
	- crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
	- crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	- crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
	-
	+ pcmk__xe_set_props(cib_node,
	+ XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES,
	+ XML_NODE_IS_PEER, ONLINESTATUS,
	+ XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER,
	+ XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER,
	+ NULL);
	} else {
	- crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
	- crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
	- crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
	- crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
	+ pcmk__xe_set_props(cib_node,
	+ XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO,
	+ XML_NODE_IS_PEER, OFFLINESTATUS,
	+ XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN,
	+ XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN,
	+ NULL);
	}
	-
	crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
	return cib_node;
	}

	static xmlNode *
	find_resource_xml(xmlNode * cib_node, const char *resource)
	{
	xmlNode *match = NULL;
	const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
	char *xpath = crm_strdup_printf(RSC_TEMPLATE, node, resource);

	match = get_xpath_object(xpath, cib_node, LOG_TRACE);
	free(xpath);
	return match;
	}

	-
	-static xmlNode *
	-inject_resource(xmlNode * cib_node, const char resource, const char lrm_name,
	- const char rclass, const char rtype, const char *rprovider)
	+/*!
	+ * \internal
	+ * \brief Inject a resource history element into a scheduler input
	+ *
	+ * \param[in] out Output object for displaying error messages
	+ * \param[in] cib_node Node state XML to inject resource history entry into
	+ * \param[in] resource ID (in configuration) of resource to inject
	+ * \param[in] lrm_name ID of resource as used in history (e.g. clone instance)
	+ * \param[in] rclass Resource agent class of resource to inject
	+ * \param[in] rtype Resource agent type of resource to inject
	+ * \param[in] rprovider Resource agent provider of resource to inject
	+ *
	+ * \return XML of injected resource history element
	+ * \note If a history element already exists under either \p resource or
	+ * \p lrm_name, this will return it rather than injecting a new one.
	+ */
	+xmlNode *
	+pcmk__inject_resource_history(pcmk__output_t out, xmlNode cib_node,
	+ const char resource, const char lrm_name,
	+ const char rclass, const char rtype,
	+ const char *rprovider)
	{
	xmlNode *lrm = NULL;
	xmlNode *container = NULL;
	xmlNode *cib_resource = NULL;
	- char *xpath = NULL;

	cib_resource = find_resource_xml(cib_node, resource);
	if (cib_resource != NULL) {
	/* If an existing LRM history entry uses the resource name,
	* continue using it, even if lrm_name is different.
	*/
	return cib_resource;
	}

	// Check for history entry under preferred name
	- if (strcmp(resource, lrm_name)) {
	+ if (strcmp(resource, lrm_name) != 0) {
	cib_resource = find_resource_xml(cib_node, lrm_name);
	if (cib_resource != NULL) {
	return cib_resource;
	}
	}

	- /* One day, add query for class, provider, type */
	-
	- if (rclass == NULL \|\| rtype == NULL) {
	+ if ((rclass == NULL) \|\| (rtype == NULL)) {
	+ // @TODO query configuration for class, provider, type
	out->err(out, "Resource %s not found in the status section of %s."
	" Please supply the class and type to continue", resource, ID(cib_node));
	return NULL;

	- } else if (!pcmk__strcase_any_of(rclass, PCMK_RESOURCE_CLASS_OCF, PCMK_RESOURCE_CLASS_STONITH,
	- PCMK_RESOURCE_CLASS_SERVICE, PCMK_RESOURCE_CLASS_UPSTART,
	- PCMK_RESOURCE_CLASS_SYSTEMD, PCMK_RESOURCE_CLASS_LSB, NULL)) {
	+ } else if (!pcmk__strcase_any_of(rclass,
	+ PCMK_RESOURCE_CLASS_OCF,
	+ PCMK_RESOURCE_CLASS_STONITH,
	+ PCMK_RESOURCE_CLASS_SERVICE,
	+ PCMK_RESOURCE_CLASS_UPSTART,
	+ PCMK_RESOURCE_CLASS_SYSTEMD,
	+ PCMK_RESOURCE_CLASS_LSB, NULL)) {
	out->err(out, "Invalid class for %s: %s", resource, rclass);
	return NULL;

	} else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
	- && (rprovider == NULL)) {
	+ && (rprovider == NULL)) {
	+ // @TODO query configuration for provider
	out->err(out, "Please specify the provider for resource %s", resource);
	return NULL;
	}

	- xpath = (char *)xmlGetNodePath(cib_node);
	- crm_info("Injecting new resource %s into %s '%s'", lrm_name, xpath, ID(cib_node));
	- free(xpath);
	+ crm_info("Injecting new resource %s into node state '%s'",
	+ lrm_name, ID(cib_node));

	lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
	if (lrm == NULL) {
	const char *node_uuid = ID(cib_node);

	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
	crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
	}

	container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
	if (container == NULL) {
	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
	}

	cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);

	// If we're creating a new entry, use the preferred name
	crm_xml_add(cib_resource, XML_ATTR_ID, lrm_name);

	crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
	crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
	crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);

	return cib_resource;
	}

	#define XPATH_MAX 1024

	static int
	-find_ticket_state(cib_t * the_cib, const char ticket_id, xmlNode * ticket_state_xml)
	+find_ticket_state(pcmk__output_t out, cib_t the_cib, const char *ticket_id,
	+ xmlNode **ticket_state_xml)
	{
	int offset = 0;
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	char *xpath_string = NULL;

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	xpath_string = calloc(1, XPATH_MAX);
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", "/cib/status/tickets");

	if (ticket_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s[@id=\"%s\"]",
	XML_CIB_TAG_TICKET_STATE, ticket_id);
	}
	CRM_LOG_ASSERT(offset > 0);
	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);

	if (rc != pcmk_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	if (ticket_id) {
	out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
	}
	*ticket_state_xml = xml_search;
	} else {
	*ticket_state_xml = xml_search;
	}

	bail:
	free(xpath_string);
	return rc;
	}

	static int
	-set_ticket_state_attr(const char ticket_id, const char attr_name,
	- const char attr_value, cib_t cib, int cib_options)
	+set_ticket_state_attr(pcmk__output_t out, const char ticket_id,
	+ const char attr_name, const char attr_value,
	+ cib_t *cib, int cib_options)
	{
	int rc = pcmk_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;

	- rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
	+ rc = find_ticket_state(out, cib, ticket_id, &ticket_state_xml);
	if (rc == pcmk_ok) {
	crm_debug("Found a match state for ticket: id=%s", ticket_id);
	xml_top = ticket_state_xml;

	} else if (rc != -ENXIO) {
	return rc;

	} else {
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
	}

	crm_xml_add(ticket_state_xml, attr_name, attr_value);

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);

	free_xml(xml_top);

	return rc;
	}

	void
	modify_configuration(pe_working_set_t * data_set, cib_t cib, pcmk_injections_t injections)
	{
	int rc = pcmk_ok;
	GList *gIter = NULL;

	xmlNode *cib_op = NULL;
	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;

	lrmd_event_data_t *op = NULL;

	- out = data_set->priv;
	+ pcmk__output_t *out = data_set->priv;

	out->message(out, "inject-modify-config", injections->quorum, injections->watchdog);

	if (injections->quorum) {
	xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);

	/* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid); */
	crm_xml_add(top, XML_ATTR_HAVE_QUORUM, injections->quorum);

	rc = cib->cmds->modify(cib, NULL, top, cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}

	if (injections->watchdog) {
	rc = update_attr_delegate(cib, cib_sync_call \| cib_scope_local,
	XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	XML_ATTR_HAVE_WATCHDOG, injections->watchdog, FALSE, NULL, NULL);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = injections->node_up; gIter != NULL; gIter = gIter->next) {
	char node = (char )gIter->data;

	out->message(out, "inject-modify-node", "Online", node);

	- cib_node = modify_node(cib, node, TRUE);
	+ cib_node = pcmk__inject_node_state_change(cib, node, true);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (gIter = injections->node_down; gIter != NULL; gIter = gIter->next) {
	char xpath[STATUS_PATH_MAX];
	char node = (char )gIter->data;

	out->message(out, "inject-modify-node", "Offline", node);

	- cib_node = modify_node(cib, node, FALSE);
	+ cib_node = pcmk__inject_node_state_change(cib, node, false);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
	XML_TAG_TRANSIENT_NODEATTRS);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	}

	for (gIter = injections->node_fail; gIter != NULL; gIter = gIter->next) {
	char node = (char )gIter->data;

	out->message(out, "inject-modify-node", "Failing", node);

	- cib_node = modify_node(cib, node, TRUE);
	+ cib_node = pcmk__inject_node_state_change(cib, node, true);
	crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (gIter = injections->ticket_grant; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	out->message(out, "inject-modify-ticket", "Granting", ticket_id);

	- rc = set_ticket_state_attr(ticket_id, "granted", "true",
	+ rc = set_ticket_state_attr(out, ticket_id, "granted", "true",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = injections->ticket_revoke; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	out->message(out, "inject-modify-ticket", "Revoking", ticket_id);

	- rc = set_ticket_state_attr(ticket_id, "granted", "false",
	+ rc = set_ticket_state_attr(out, ticket_id, "granted", "false",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = injections->ticket_standby; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	out->message(out, "inject-modify-ticket", "Standby", ticket_id);

	- rc = set_ticket_state_attr(ticket_id, "standby", "true",
	+ rc = set_ticket_state_attr(out, ticket_id, "standby", "true",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = injections->ticket_activate; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	out->message(out, "inject-modify-ticket", "Activating", ticket_id);

	- rc = set_ticket_state_attr(ticket_id, "standby", "false",
	+ rc = set_ticket_state_attr(out, ticket_id, "standby", "false",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = injections->op_inject; gIter != NULL; gIter = gIter->next) {
	char spec = (char )gIter->data;

	int rc = 0;
	int outcome = PCMK_OCF_OK;
	guint interval_ms = 0;

	char *key = NULL;
	char *node = NULL;
	char *task = NULL;
	char *resource = NULL;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *rprovider = NULL;

	pe_resource_t *rsc = NULL;

	out->message(out, "inject-spec", spec);

	key = calloc(1, strlen(spec) + 1);
	node = calloc(1, strlen(spec) + 1);
	rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
	if (rc != 3) {
	out->err(out, "Invalid operation spec: %s. Only found %d fields", spec, rc);
	free(key);
	free(node);
	continue;
	}

	parse_op_key(key, &resource, &task, &interval_ms);

	rsc = pe_find_resource(data_set->resources, resource);
	if (rsc == NULL) {
	out->err(out, "Invalid resource name: %s", resource);
	} else {
	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	- cib_node = inject_node_state(cib, node, NULL);
	+ cib_node = pcmk__inject_node(cib, node, NULL);
	CRM_ASSERT(cib_node != NULL);

	- update_failcounts(cib_node, resource, task, interval_ms, outcome);
	+ pcmk__inject_failcount(out, cib_node, resource, task, interval_ms,
	+ outcome);

	- cib_resource = inject_resource(cib_node, resource, resource,
	- rclass, rtype, rprovider);
	+ cib_resource = pcmk__inject_resource_history(out, cib_node,
	+ resource, resource,
	+ rclass, rtype,
	+ rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = create_op(cib_resource, task, interval_ms, outcome);
	CRM_ASSERT(op != NULL);

	- cib_op = inject_op(cib_resource, op, 0);
	+ cib_op = pcmk__inject_action_result(cib_resource, op, 0);
	CRM_ASSERT(cib_op != NULL);
	lrmd_free_event(op);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}
	free(task);
	free(node);
	free(key);
	}

	if (!out->is_quiet(out)) {
	out->end_list(out);
	}
	}
	-
	-static gboolean
	-exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
	-{
	- const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	- const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	-
	- crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	- out->message(out, "inject-pseudo-action", node, task);
	-
	- pcmk__update_graph(graph, action);
	- return TRUE;
	-}
	-
	-static gboolean
	-exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
	-{
	- int rc = 0;
	- GList *gIter = NULL;
	- lrmd_event_data_t *op = NULL;
	- int target_outcome = PCMK_OCF_OK;
	-
	- const char *rtype = NULL;
	- const char *rclass = NULL;
	- const char *resource = NULL;
	- const char *rprovider = NULL;
	- const char *lrm_name = NULL;
	- const char *operation = crm_element_value(action->xml, "operation");
	- const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
	-
	- xmlNode *cib_node = NULL;
	- xmlNode *cib_resource = NULL;
	- xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
	-
	- char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	- char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	- const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
	-
	- if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	- crm_info("Skipping %s op for %s", operation, node);
	- goto done;
	- }
	-
	- if (action_rsc == NULL) {
	- crm_log_xml_err(action->xml, "Bad");
	- free(node); free(uuid);
	- return FALSE;
	- }
	-
	- /* Look for the preferred name
	- * If not found, try the expected 'local' name
	- * If not found use the preferred name anyway
	- */
	- resource = crm_element_value(action_rsc, XML_ATTR_ID);
	- CRM_ASSERT(resource != NULL); // makes static analysis happy
	- lrm_name = resource; // Preferred name when writing history
	- if (pe_find_resource(fake_resource_list, resource) == NULL) {
	- const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
	-
	- if (longname && pe_find_resource(fake_resource_list, longname)) {
	- resource = longname;
	- }
	- }
	-
	- if (pcmk__strcase_any_of(operation, "delete", RSC_METADATA, NULL)) {
	- out->message(out, "inject-rsc-action", resource, operation, node, (guint) 0);
	- goto done;
	- }
	-
	- rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	- rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
	- rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
	-
	- pcmk__scan_min_int(target_rc_s, &target_outcome, 0);
	-
	- CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call \| cib_scope_local) ==
	- pcmk_ok);
	-
	- cib_node = inject_node_state(fake_cib, node, (router_node? node : uuid));
	- CRM_ASSERT(cib_node != NULL);
	-
	- cib_resource = inject_resource(cib_node, resource, lrm_name,
	- rclass, rtype, rprovider);
	- if (cib_resource == NULL) {
	- crm_err("invalid resource in transition");
	- free(node); free(uuid);
	- free_xml(cib_node);
	- return FALSE;
	- }
	-
	- op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	- target_outcome, "User-injected result");
	-
	- out->message(out, "inject-rsc-action", resource, op->op_type, node, op->interval_ms);
	-
	- for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
	- char spec = (char )gIter->data;
	- char *key = NULL;
	- const char *match_name = NULL;
	-
	- // Allow user to specify anonymous clone with or without instance number
	- key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	- op->interval_ms, node);
	- if (strncasecmp(key, spec, strlen(key)) == 0) {
	- match_name = resource;
	- }
	- free(key);
	-
	- if ((match_name == NULL) && strcmp(resource, lrm_name)) {
	- key = crm_strdup_printf(PCMK__OP_FMT "@%s=", lrm_name, op->op_type,
	- op->interval_ms, node);
	- if (strncasecmp(key, spec, strlen(key)) == 0) {
	- match_name = lrm_name;
	- }
	- free(key);
	- }
	-
	- if (match_name != NULL) {
	-
	- rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	- // ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	-
	- if (rc != 1) {
	- out->err(out,
	- "Invalid failed operation spec: %s. Result code must be integer",
	- spec);
	- continue;
	- }
	- crm__set_graph_action_flags(action, pcmk__graph_action_failed);
	- graph->abort_priority = INFINITY;
	- out->info(out, "Pretending action %d failed with rc=%d", action->id, op->rc);
	- update_failcounts(cib_node, match_name, op->op_type,
	- op->interval_ms, op->rc);
	- break;
	- }
	- }
	-
	- inject_op(cib_resource, op, target_outcome);
	- lrmd_free_event(op);
	-
	- rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	- cib_sync_call \| cib_scope_local);
	- CRM_ASSERT(rc == pcmk_ok);
	-
	- done:
	- free(node); free(uuid);
	- free_xml(cib_node);
	- crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	- pcmk__update_graph(graph, action);
	- return TRUE;
	-}
	-
	-static gboolean
	-exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
	-{
	- const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	- const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	- xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
	-
	- crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	- out->message(out, "inject-cluster-action", node, task, rsc);
	- pcmk__update_graph(graph, action);
	- return TRUE;
	-}
	-
	-static gboolean
	-exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
	-{
	- const char *op = crm_meta_value(action->params, "stonith_action");
	- char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	-
	- out->message(out, "inject-fencing-action", target, op);
	-
	- if(!pcmk__str_eq(op, "on", pcmk__str_casei)) {
	- int rc = 0;
	- char xpath[STATUS_PATH_MAX];
	- xmlNode *cib_node = modify_node(fake_cib, target, FALSE);
	-
	- crm_xml_add(cib_node, XML_ATTR_ORIGIN, __func__);
	- CRM_ASSERT(cib_node != NULL);
	-
	- rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	- cib_sync_call \| cib_scope_local);
	- CRM_ASSERT(rc == pcmk_ok);
	-
	- snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
	- fake_cib->cmds->remove(fake_cib, xpath, NULL,
	- cib_xpath \| cib_sync_call \| cib_scope_local);
	-
	- snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
	- XML_TAG_TRANSIENT_NODEATTRS);
	- fake_cib->cmds->remove(fake_cib, xpath, NULL,
	- cib_xpath \| cib_sync_call \| cib_scope_local);
	-
	- free_xml(cib_node);
	- }
	-
	- crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	- pcmk__update_graph(graph, action);
	- free(target);
	- return TRUE;
	-}
	-
	-enum transition_status
	-run_simulation(pe_working_set_t * data_set, cib_t cib, GList op_fail_list)
	-{
	- crm_graph_t *transition = NULL;
	- enum transition_status graph_rc;
	-
	- crm_graph_functions_t exec_fns = {
	- exec_pseudo_action,
	- exec_rsc_action,
	- exec_crmd_action,
	- exec_stonith_action,
	- };
	-
	- out = data_set->priv;
	-
	- fake_cib = cib;
	- fake_op_fail_list = op_fail_list;
	-
	- if (!out->is_quiet(out)) {
	- out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	- }
	-
	- pcmk__set_graph_functions(&exec_fns);
	- transition = pcmk__unpack_graph(data_set->graph, crm_system_name);
	- pcmk__log_graph(LOG_DEBUG, transition);
	-
	- fake_resource_list = data_set->resources;
	- do {
	- graph_rc = pcmk__execute_graph(transition);
	-
	- } while (graph_rc == transition_active);
	- fake_resource_list = NULL;
	-
	- if (graph_rc != transition_complete) {
	- out->err(out, "Transition failed: %s",
	- pcmk__graph_status2text(graph_rc));
	- pcmk__log_graph(LOG_ERR, transition);
	- }
	- pcmk__free_graph(transition);
	- if (graph_rc != transition_complete) {
	- out->err(out, "An invalid transition was produced");
	- }
	-
	- if (!out->is_quiet(out)) {
	- xmlNode *cib_object = NULL;
	- int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call \| cib_scope_local);
	-
	- CRM_ASSERT(rc == pcmk_ok);
	- pe_reset_working_set(data_set);
	- data_set->input = cib_object;
	-
	- out->end_list(out);
	- }
	-
	- return graph_rc;
	-}
	diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
	index d97134d15a..f144b8822a 100644
	--- a/lib/pacemaker/pcmk_simulate.c
	+++ b/lib/pacemaker/pcmk_simulate.c
	@@ -1,560 +1,991 @@
	/*
	* Copyright 2021-2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/pengine/pe_types.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	+#define STATUS_PATH_MAX 512
	+
	+static pcmk__output_t *out = NULL;
	+static cib_t *fake_cib = NULL;
	+static GList *fake_resource_list = NULL;
	+static GList *fake_op_fail_list = NULL;
	+
	+static void set_effective_date(pe_working_set_t *data_set, bool print_original,
	+ char *use_date);
	+
	+/*!
	+ * \internal
	+ * \brief Create an action name for use in a dot graph
	+ *
	+ * \param[in] action Action to create name for
	+ * \param[in] verbose If true, add action ID to name
	+ *
	+ * \return Newly allocated string with action name
	+ * \note It is the caller's responsibility to free the result.
	+ */
	static char *
	create_action_name(pe_action_t *action, bool verbose)
	{
	char *action_name = NULL;
	const char *prefix = "";
	const char *action_host = NULL;
	const char *clone_name = NULL;
	const char *task = action->task;

	- if (action->node) {
	+ if (action->node != NULL) {
	action_host = action->node->details->uname;
	} else if (!pcmk_is_set(action->flags, pe_action_pseudo)) {
	action_host = "<none>";
	}

	- if (pcmk__str_eq(action->task, RSC_CANCEL, pcmk__str_casei)) {
	+ if (pcmk__str_eq(action->task, RSC_CANCEL, pcmk__str_none)) {
	prefix = "Cancel ";
	task = action->cancel_task;
	}

	- if (action->rsc && action->rsc->clone_name) {
	+ if (action->rsc != NULL) {
	clone_name = action->rsc->clone_name;
	}

	- if (clone_name) {
	+ if (clone_name != NULL) {
	char *key = NULL;
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	- if (pcmk__strcase_any_of(action->task, RSC_NOTIFY, RSC_NOTIFIED, NULL)) {
	- const char *n_type = g_hash_table_lookup(action->meta, "notify_key_type");
	- const char *n_task = g_hash_table_lookup(action->meta, "notify_key_operation");
	+ if (pcmk__strcase_any_of(action->task, RSC_NOTIFY, RSC_NOTIFIED,
	+ NULL)) {
	+ const char *n_type = g_hash_table_lookup(action->meta,
	+ "notify_key_type");
	+ const char *n_task = g_hash_table_lookup(action->meta,
	+ "notify_key_operation");

	CRM_ASSERT(n_type != NULL);
	CRM_ASSERT(n_task != NULL);
	key = pcmk__notify_key(clone_name, n_type, n_task);
	-
	} else {
	key = pcmk__op_key(clone_name, task, interval_ms);
	}

	- if (action_host) {
	- action_name = crm_strdup_printf("%s%s %s", prefix, key, action_host);
	+ if (action_host != NULL) {
	+ action_name = crm_strdup_printf("%s%s %s",
	+ prefix, key, action_host);
	} else {
	action_name = crm_strdup_printf("%s%s", prefix, key);
	}
	free(key);

	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	- action_name = crm_strdup_printf("%s%s '%s' %s", prefix, action->task, op, action_host);
	+ action_name = crm_strdup_printf("%s%s '%s' %s",
	+ prefix, action->task, op, action_host);

	} else if (action->rsc && action_host) {
	- action_name = crm_strdup_printf("%s%s %s", prefix, action->uuid, action_host);
	+ action_name = crm_strdup_printf("%s%s %s",
	+ prefix, action->uuid, action_host);

	} else if (action_host) {
	- action_name = crm_strdup_printf("%s%s %s", prefix, action->task, action_host);
	+ action_name = crm_strdup_printf("%s%s %s",
	+ prefix, action->task, action_host);

	} else {
	action_name = crm_strdup_printf("%s", action->uuid);
	}

	if (verbose) {
	char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);

	free(action_name);
	action_name = with_id;
	}
	return action_name;
	}

	+/*!
	+ * \internal
	+ * \brief Display the status of a cluster
	+ *
	+ * \param[in] data_set Cluster working set
	+ * \param[in] show_opts How to modify display (as pcmk_show_opt_e flags)
	+ * \param[in] section_opts Sections to display (as pcmk_section_e flags)
	+ * \param[in] title What to use as list title
	+ * \param[in] print_spacer Whether to display a spacer first
	+ */
	static void
	-print_cluster_status(pe_working_set_t * data_set, unsigned int show_opts,
	- unsigned int section_opts, const char *title, bool print_spacer)
	+print_cluster_status(pe_working_set_t *data_set, unsigned int show_opts,
	+ unsigned int section_opts, const char *title,
	+ bool print_spacer)
	{
	pcmk__output_t *out = data_set->priv;
	GList *all = NULL;

	section_opts \|= pcmk_section_nodes \| pcmk_section_resources;

	all = g_list_prepend(all, (gpointer) "*");

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "%s", title);
	out->message(out, "cluster-status", data_set, 0, NULL, FALSE,
	section_opts,
	show_opts \| pcmk_show_inactive_rscs \| pcmk_show_failed_detail,
	NULL, all, all);
	out->end_list(out);

	g_list_free(all);
	}

	+/*!
	+ * \internal
	+ * \brief Display a summary of all actions scheduled in a transition
	+ *
	+ * \param[in] data_set Cluster working set (fully scheduled)
	+ * \param[in] print_spacer Whether to display a spacer first
	+ */
	static void
	print_transition_summary(pe_working_set_t *data_set, bool print_spacer)
	{
	pcmk__output_t *out = data_set->priv;

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "Transition Summary");
	pcmk__output_actions(data_set);
	out->end_list(out);
	}

	+/*!
	+ * \internal
	+ * \brief Reset a cluster working set's input, output, date, and flags
	+ *
	+ * \param[in] data_set Cluster working set
	+ * \param[in] input What to set as cluster input
	+ * \param[in] out What to set as cluster output object
	+ * \param[in] use_date What to set as cluster's current timestamp
	+ * \param[in] flags Cluster flags to add (pe_flag_*)
	+ */
	static void
	reset(pe_working_set_t data_set, xmlNodePtr input, pcmk__output_t out,
	char *use_date, unsigned int flags)
	{
	data_set->input = input;
	data_set->priv = out;
	- pcmk__set_effective_date(data_set, true, use_date);
	+ set_effective_date(data_set, true, use_date);
	if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
	pe__set_working_set_flags(data_set, pe_flag_sanitized);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pe__set_working_set_flags(data_set, pe_flag_show_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	}
	}

	-int
	-pcmk__write_sim_dotfile(pe_working_set_t data_set, const char dot_file,
	- bool all_actions, bool verbose)
	+/*!
	+ * \brief Write out a file in dot(1) format describing the actions that will
	+ * be taken by the scheduler in response to an input CIB file.
	+ *
	+ * \param[in] data_set Working set for the cluster
	+ * \param[in] dot_file The filename to write
	+ * \param[in] all_actions Write all actions, even those that are optional or
	+ * are on unmanaged resources
	+ * \param[in] verbose Add extra information, such as action IDs, to the
	+ * output
	+ *
	+ * \return Standard Pacemaker return code
	+ */
	+static int
	+write_sim_dotfile(pe_working_set_t data_set, const char dot_file,
	+ bool all_actions, bool verbose)
	{
	GList *gIter = NULL;
	FILE *dot_strm = fopen(dot_file, "w");

	if (dot_strm == NULL) {
	return errno;
	}

	fprintf(dot_strm, " digraph \"g\" {\n");
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	const char *style = "dashed";
	const char *font = "black";
	const char *color = "black";
	char *action_name = create_action_name(action, verbose);

	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	font = "orange";
	}

	if (pcmk_is_set(action->flags, pe_action_dumped)) {
	style = "bold";
	color = "green";

	} else if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
	color = "red";
	font = "purple";
	if (!all_actions) {
	goto do_not_write;
	}

	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	color = "blue";
	if (!all_actions) {
	goto do_not_write;
	}

	} else {
	color = "red";
	CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pe_action_runnable));
	}

	pe__set_action_flags(action, pe_action_dumped);
	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
	action_name, style, color, font);
	do_not_write:
	free(action_name);
	}

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	GList *gIter2 = NULL;

	for (gIter2 = action->actions_before; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_action_wrapper_t before = (pe_action_wrapper_t ) gIter2->data;

	char *before_name = NULL;
	char *after_name = NULL;
	const char *style = "dashed";
	bool optional = true;

	if (before->state == pe_link_dumped) {
	optional = false;
	style = "bold";
	} else if (before->type == pe_order_none) {
	continue;
	} else if (pcmk_is_set(before->action->flags, pe_action_dumped)
	&& pcmk_is_set(action->flags, pe_action_dumped)
	&& before->type != pe_order_load) {
	optional = false;
	}

	if (all_actions \|\| !optional) {
	before_name = create_action_name(before->action, verbose);
	after_name = create_action_name(action, verbose);
	fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
	before_name, after_name, style);
	free(before_name);
	free(after_name);
	}
	}
	}

	fprintf(dot_strm, "}\n");
	fflush(dot_strm);
	fclose(dot_strm);
	return pcmk_rc_ok;
	}

	-void
	-pcmk__profile_file(const char xml_file, long long repeat, pe_working_set_t data_set, char *use_date)
	+/*!
	+ * \brief Profile the configuration updates and scheduler actions in a single
	+ * CIB file, printing the profiling timings.
	+ *
	+ * \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	+ * object before this function is called.
	+ *
	+ * \param[in] xml_file The CIB file to profile
	+ * \param[in] repeat Number of times to run
	+ * \param[in] data_set Working set for the cluster
	+ * \param[in] use_date The date to set the cluster's time to (may be NULL)
	+ */
	+static void
	+profile_file(const char xml_file, long long repeat, pe_working_set_t data_set,
	+ char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	xmlNode *cib_object = NULL;
	clock_t start = 0;
	clock_t end;

	CRM_ASSERT(out != NULL);

	cib_object = filename2xml(xml_file);
	start = clock();

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return;
	}

	for (int i = 0; i < repeat; ++i) {
	xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);

	data_set->input = input;
	- pcmk__set_effective_date(data_set, false, use_date);
	+ set_effective_date(data_set, false, use_date);
	pcmk__schedule_actions(data_set, input, NULL);
	pe_reset_working_set(data_set);
	}

	end = clock();
	out->message(out, "profile", xml_file, start, end);
	}

	void
	pcmk__profile_dir(const char dir, long long repeat, pe_working_set_t data_set, char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	struct dirent **namelist;

	int file_num = scandir(dir, &namelist, 0, alphasort);

	CRM_ASSERT(out != NULL);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX];

	out->begin_list(out, NULL, NULL, "Timings");

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
	".xml")) {
	free(namelist[file_num]);
	continue;
	}
	snprintf(buffer, sizeof(buffer), "%s/%s", dir, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	- pcmk__profile_file(buffer, repeat, data_set, use_date);
	+ profile_file(buffer, repeat, data_set, use_date);
	}
	free(namelist[file_num]);
	}
	free(namelist);

	out->end_list(out);
	}
	}

	-void
	-pcmk__set_effective_date(pe_working_set_t data_set, bool print_original, char use_date)
	+/*!
	+ * \brief Set the date of the cluster, either to the value given by
	+ * \p use_date, or to the "execution-date" value in the CIB.
	+ *
	+ * \note \p data_set->priv must have been set to a valid \p pcmk__output_t
	+ * object before this function is called.
	+ *
	+ * \param[in,out] data_set Working set for the cluster
	+ * \param[in] print_original If \p true, the "execution-date" should
	+ * also be printed
	+ * \param[in] use_date The date to set the cluster's time to
	+ * (may be NULL)
	+ */
	+static void
	+set_effective_date(pe_working_set_t *data_set, bool print_original,
	+ char *use_date)
	{
	pcmk__output_t *out = data_set->priv;
	time_t original_date = 0;

	CRM_ASSERT(out != NULL);

	crm_element_value_epoch(data_set->input, "execution-date", &original_date);

	if (use_date) {
	data_set->now = crm_time_new(use_date);
	out->info(out, "Setting effective cluster time: %s", use_date);
	crm_time_log(LOG_NOTICE, "Pretending 'now' is", data_set->now,
	crm_time_log_date \| crm_time_log_timeofday);

	} else if (original_date) {

	data_set->now = crm_time_new(NULL);
	crm_time_set_timet(data_set->now, &original_date);

	if (print_original) {
	char *when = crm_time_as_string(data_set->now,
	crm_time_log_date\|crm_time_log_timeofday);

	out->info(out, "Using the original execution date of: %s", when);
	free(when);
	}
	}
	}

	+/*!
	+ * \internal
	+ * \brief Simulate successfully executing a pseudo-action in a graph
	+ *
	+ * \param[in] graph Graph to update with pseudo-action result
	+ * \param[in] action Pseudo-action to simulate executing
	+ *
	+ * \return TRUE
	+ */
	+static gboolean
	+simulate_pseudo_action(crm_graph_t graph, crm_action_t action)
	+{
	+ const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	+ const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	+
	+ crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	+ out->message(out, "inject-pseudo-action", node, task);
	+
	+ pcmk__update_graph(graph, action);
	+ return TRUE;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Simulate executing a resource action in a graph
	+ *
	+ * \param[in] graph Graph to update with resource action result
	+ * \param[in] action Resource action to simulate executing
	+ *
	+ * \return TRUE if action is validly specified, otherwise FALSE
	+ */
	+static gboolean
	+simulate_resource_action(crm_graph_t graph, crm_action_t action)
	+{
	+ int rc;
	+ lrmd_event_data_t *op = NULL;
	+ int target_outcome = PCMK_OCF_OK;
	+
	+ const char *rtype = NULL;
	+ const char *rclass = NULL;
	+ const char *resource = NULL;
	+ const char *rprovider = NULL;
	+ const char *resource_config_name = NULL;
	+ const char *operation = crm_element_value(action->xml, "operation");
	+ const char *target_rc_s = crm_meta_value(action->params,
	+ XML_ATTR_TE_TARGET_RC);
	+
	+ xmlNode *cib_node = NULL;
	+ xmlNode *cib_resource = NULL;
	+ xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
	+
	+ char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	+ char *uuid = NULL;
	+ const char *router_node = crm_element_value(action->xml,
	+ XML_LRM_ATTR_ROUTER_NODE);
	+
	+ // Certain actions don't need to be displayed or history entries
	+ if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	+ crm_debug("No history injection for %s op on %s", operation, node);
	+ goto done; // Confirm action and update graph
	+ }
	+
	+ if (action_rsc == NULL) { // Shouldn't be possible
	+ crm_log_xml_err(action->xml, "Bad");
	+ free(node);
	+ return FALSE;
	+ }
	+
	+ /* A resource might be known by different names in the configuration and in
	+ * the action (for example, a clone instance). Grab the configuration name
	+ * (which is preferred when writing history), and if necessary, the instance
	+ * name.
	+ */
	+ resource_config_name = crm_element_value(action_rsc, XML_ATTR_ID);
	+ if (resource_config_name == NULL) { // Shouldn't be possible
	+ crm_log_xml_err(action->xml, "No ID");
	+ free(node);
	+ return FALSE;
	+ }
	+ resource = resource_config_name;
	+ if (pe_find_resource(fake_resource_list, resource) == NULL) {
	+ const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
	+
	+ if ((longname != NULL)
	+ && (pe_find_resource(fake_resource_list, longname) != NULL)) {
	+ resource = longname;
	+ }
	+ }
	+
	+ // Certain actions need to be displayed but don't need history entries
	+ if (pcmk__strcase_any_of(operation, "delete", RSC_METADATA, NULL)) {
	+ out->message(out, "inject-rsc-action", resource, operation, node,
	+ (guint) 0);
	+ goto done; // Confirm action and update graph
	+ }
	+
	+ rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	+ rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
	+ rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
	+
	+ pcmk__scan_min_int(target_rc_s, &target_outcome, 0);
	+
	+ CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
	+ cib_sync_call\|cib_scope_local) == pcmk_ok);
	+
	+ // Ensure the action node is in the CIB
	+ uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	+ cib_node = pcmk__inject_node(fake_cib, node,
	+ ((router_node == NULL)? uuid: node));
	+ free(uuid);
	+ CRM_ASSERT(cib_node != NULL);
	+
	+ // Add a history entry for the action
	+ cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
	+ resource_config_name,
	+ rclass, rtype, rprovider);
	+ if (cib_resource == NULL) {
	+ crm_err("Could not simulate action %d history for resource %s",
	+ action->id, resource);
	+ free(node);
	+ free_xml(cib_node);
	+ return FALSE;
	+ }
	+
	+ // Simulate and display an executor event for the action result
	+ op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	+ target_outcome, "User-injected result");
	+ out->message(out, "inject-rsc-action", resource, op->op_type, node,
	+ op->interval_ms);
	+
	+ // Check whether action is in a list of desired simulated failures
	+ for (GList *iter = fake_op_fail_list; iter != NULL; iter = iter->next) {
	+ char spec = (char ) iter->data;
	+ char *key = NULL;
	+ const char *match_name = NULL;
	+
	+ // Allow user to specify anonymous clone with or without instance number
	+ key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	+ op->interval_ms, node);
	+ if (strncasecmp(key, spec, strlen(key)) == 0) {
	+ match_name = resource;
	+ }
	+ free(key);
	+
	+ // If not found, try the resource's name in the configuration
	+ if ((match_name == NULL)
	+ && (strcmp(resource, resource_config_name) != 0)) {
	+
	+ key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
	+ op->op_type, op->interval_ms, node);
	+ if (strncasecmp(key, spec, strlen(key)) == 0) {
	+ match_name = resource_config_name;
	+ }
	+ free(key);
	+ }
	+
	+ if (match_name == NULL) {
	+ continue; // This failed action entry doesn't match
	+ }
	+
	+ // ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	+ rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	+ if (rc != 1) {
	+ out->err(out, "Invalid failed operation '%s' "
	+ "(result code must be integer)", spec);
	+ continue; // Keep checking other list entries
	+ }
	+
	+ out->info(out, "Pretending action %d failed with rc=%d",
	+ action->id, op->rc);
	+ crm__set_graph_action_flags(action, pcmk__graph_action_failed);
	+ graph->abort_priority = INFINITY;
	+ pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
	+ op->interval_ms, op->rc);
	+ break;
	+ }
	+
	+ pcmk__inject_action_result(cib_resource, op, target_outcome);
	+ lrmd_free_event(op);
	+ rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	+ cib_sync_call\|cib_scope_local);
	+ CRM_ASSERT(rc == pcmk_ok);
	+
	+ done:
	+ free(node);
	+ free_xml(cib_node);
	+ crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	+ pcmk__update_graph(graph, action);
	+ return TRUE;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Simulate successfully executing a cluster action
	+ *
	+ * \param[in] graph Graph to update with action result
	+ * \param[in] action Cluster action to simulate
	+ *
	+ * \return TRUE
	+ */
	+static gboolean
	+simulate_cluster_action(crm_graph_t graph, crm_action_t action)
	+{
	+ const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	+ const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	+ xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
	+
	+ crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	+ out->message(out, "inject-cluster-action", node, task, rsc);
	+ pcmk__update_graph(graph, action);
	+ return TRUE;
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Simulate successfully executing a fencing action
	+ *
	+ * \param[in] graph Graph to update with action result
	+ * \param[in] action Fencing action to simulate
	+ *
	+ * \return TRUE
	+ */
	+static gboolean
	+simulate_fencing_action(crm_graph_t graph, crm_action_t action)
	+{
	+ const char *op = crm_meta_value(action->params, "stonith_action");
	+ char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	+
	+ out->message(out, "inject-fencing-action", target, op);
	+
	+ if (!pcmk__str_eq(op, "on", pcmk__str_casei)) {
	+ int rc = pcmk_ok;
	+ char xpath[STATUS_PATH_MAX];
	+
	+ // Set node state to offline
	+ xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
	+ false);
	+
	+ CRM_ASSERT(cib_node != NULL);
	+ crm_xml_add(cib_node, XML_ATTR_ORIGIN, __func__);
	+ rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	+ cib_sync_call\|cib_scope_local);
	+ CRM_ASSERT(rc == pcmk_ok);
	+
	+ // Simulate controller clearing node's resource history and attributes
	+ snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s",
	+ target, XML_CIB_TAG_LRM);
	+ fake_cib->cmds->remove(fake_cib, xpath, NULL,
	+ cib_xpath\|cib_sync_call\|cib_scope_local);
	+
	+ snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s",
	+ target, XML_TAG_TRANSIENT_NODEATTRS);
	+ fake_cib->cmds->remove(fake_cib, xpath, NULL,
	+ cib_xpath\|cib_sync_call\|cib_scope_local);
	+
	+ free_xml(cib_node);
	+ }
	+
	+ crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	+ pcmk__update_graph(graph, action);
	+ free(target);
	+ return TRUE;
	+}
	+
	+enum transition_status
	+pcmk__simulate_transition(pe_working_set_t data_set, cib_t cib,
	+ GList *op_fail_list)
	+{
	+ crm_graph_t *transition = NULL;
	+ enum transition_status graph_rc;
	+
	+ crm_graph_functions_t simulation_fns = {
	+ simulate_pseudo_action,
	+ simulate_resource_action,
	+ simulate_cluster_action,
	+ simulate_fencing_action,
	+ };
	+
	+ out = data_set->priv;
	+
	+ fake_cib = cib;
	+ fake_op_fail_list = op_fail_list;
	+
	+ if (!out->is_quiet(out)) {
	+ out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	+ }
	+
	+ pcmk__set_graph_functions(&simulation_fns);
	+ transition = pcmk__unpack_graph(data_set->graph, crm_system_name);
	+ pcmk__log_graph(LOG_DEBUG, transition);
	+
	+ fake_resource_list = data_set->resources;
	+ do {
	+ graph_rc = pcmk__execute_graph(transition);
	+ } while (graph_rc == transition_active);
	+ fake_resource_list = NULL;
	+
	+ if (graph_rc != transition_complete) {
	+ out->err(out, "Transition failed: %s",
	+ pcmk__graph_status2text(graph_rc));
	+ pcmk__log_graph(LOG_ERR, transition);
	+ out->err(out, "An invalid transition was produced");
	+ }
	+ pcmk__free_graph(transition);
	+
	+ if (!out->is_quiet(out)) {
	+ // If not quiet, we'll need the resulting CIB for later display
	+ xmlNode *cib_object = NULL;
	+ int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
	+ cib_sync_call\|cib_scope_local);
	+
	+ CRM_ASSERT(rc == pcmk_ok);
	+ pe_reset_working_set(data_set);
	+ data_set->input = cib_object;
	+ out->end_list(out);
	+ }
	+ return graph_rc;
	+}
	+
	int
	-pcmk__simulate(pe_working_set_t data_set, pcmk__output_t out, pcmk_injections_t *injections,
	- unsigned int flags, unsigned int section_opts, char *use_date,
	- char input_file, char graph_file, char *dot_file)
	+pcmk__simulate(pe_working_set_t data_set, pcmk__output_t out,
	+ pcmk_injections_t *injections, unsigned int flags,
	+ unsigned int section_opts, char use_date, char input_file,
	+ char graph_file, char dot_file)
	{
	int printed = pcmk_rc_no_output;
	int rc = pcmk_rc_ok;
	xmlNodePtr input = NULL;
	cib_t *cib = NULL;
	- bool modified = false;

	rc = cib__signon_query(&cib, &input);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}

	reset(data_set, input, out, use_date, flags);
	cluster_status(data_set);

	if (!out->is_quiet(out)) {
	if (pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
	printed = out->message(out, "maint-mode", data_set->flags);
	}

	if (data_set->disabled_resources \|\| data_set->blocked_resources) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	- printed = out->info(out, "%d of %d resource instances DISABLED and %d BLOCKED "
	- "from further action due to failure",
	- data_set->disabled_resources, data_set->ninstances,
	+ printed = out->info(out,
	+ "%d of %d resource instances DISABLED and "
	+ "%d BLOCKED from further action due to failure",
	+ data_set->disabled_resources,
	+ data_set->ninstances,
	data_set->blocked_resources);
	}

	/* Most formatted output headers use caps for each word, but this one
	* only has the first word capitalized for compatibility with pcs.
	*/
	- print_cluster_status(data_set, pcmk_is_set(flags, pcmk_sim_show_pending) ? pcmk_show_pending : 0,
	- section_opts, "Current cluster status", printed == pcmk_rc_ok);
	+ print_cluster_status(data_set,
	+ pcmk_is_set(flags, pcmk_sim_show_pending)? pcmk_show_pending : 0,
	+ section_opts, "Current cluster status",
	+ (printed == pcmk_rc_ok));
	printed = pcmk_rc_ok;
	}

	- modified = injections->node_down != NULL \|\| injections->node_fail != NULL \|\|
	- injections->node_up != NULL \|\| injections->op_inject != NULL \|\|
	- injections->ticket_activate != NULL \|\| injections->ticket_grant != NULL \|\|
	- injections->ticket_revoke != NULL \|\| injections->ticket_standby != NULL \|\|
	- injections->watchdog != NULL \|\| injections->watchdog != NULL;
	+ // If the user requested any injections, handle them
	+ if ((injections->node_down != NULL)
	+ \|\| (injections->node_fail != NULL)
	+ \|\| (injections->node_up != NULL)
	+ \|\| (injections->op_inject != NULL)
	+ \|\| (injections->ticket_activate != NULL)
	+ \|\| (injections->ticket_grant != NULL)
	+ \|\| (injections->ticket_revoke != NULL)
	+ \|\| (injections->ticket_standby != NULL)
	+ \|\| (injections->watchdog != NULL)) {

	- if (modified) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	modify_configuration(data_set, cib, injections);
	printed = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
	if (rc != pcmk_rc_ok) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}

	cleanup_calculations(data_set);
	reset(data_set, input, out, use_date, flags);
	cluster_status(data_set);
	}

	if (input_file != NULL) {
	rc = write_xml_file(input, input_file, FALSE);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}
	}

	if (pcmk_any_flags_set(flags, pcmk_sim_process \| pcmk_sim_simulate)) {
	crm_time_t *local_date = NULL;
	pcmk__output_t *logger_out = NULL;

	- if (pcmk_all_flags_set(data_set->flags, pe_flag_show_scores\|pe_flag_show_utilization)) {
	+ if (pcmk_all_flags_set(data_set->flags,
	+ pe_flag_show_scores\|pe_flag_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	- out->begin_list(out, NULL, NULL, "Allocation Scores and Utilization Information");
	+ out->begin_list(out, NULL, NULL,
	+ "Allocation Scores and Utilization Information");
	printed = pcmk_rc_ok;
	+
	} else if (pcmk_is_set(data_set->flags, pe_flag_show_scores)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Allocation Scores");
	printed = pcmk_rc_ok;
	+
	} else if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Utilization Information");
	printed = pcmk_rc_ok;
	+
	} else {
	logger_out = pcmk__new_logger();
	if (logger_out == NULL) {
	rc = pcmk_rc_error;
	goto simulate_done;
	}
	-
	data_set->priv = logger_out;
	}

	pcmk__schedule_actions(data_set, input, local_date);

	if (logger_out == NULL) {
	out->end_list(out);
	} else {
	logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger_out);
	data_set->priv = out;
	}

	input = NULL; /* Don't try and free it twice */

	if (graph_file != NULL) {
	rc = write_xml_file(data_set->graph, graph_file, FALSE);
	if (rc < 0) {
	rc = pcmk_rc_graph_error;
	goto simulate_done;
	}
	}

	if (dot_file != NULL) {
	- rc = pcmk__write_sim_dotfile(data_set, dot_file,
	- pcmk_is_set(flags, pcmk_sim_all_actions),
	- pcmk_is_set(flags, pcmk_sim_verbose));
	+ rc = write_sim_dotfile(data_set, dot_file,
	+ pcmk_is_set(flags, pcmk_sim_all_actions),
	+ pcmk_is_set(flags, pcmk_sim_verbose));
	if (rc != pcmk_rc_ok) {
	rc = pcmk_rc_dot_error;
	goto simulate_done;
	}
	}

	if (!out->is_quiet(out)) {
	print_transition_summary(data_set, printed == pcmk_rc_ok);
	}
	}

	rc = pcmk_rc_ok;

	- if (pcmk_is_set(flags, pcmk_sim_simulate)) {
	- PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	- if (run_simulation(data_set, cib, injections->op_fail) != transition_complete) {
	- rc = pcmk_rc_invalid_transition;
	- }
	+ if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
	+ goto simulate_done;
	+ }

	- if (!out->is_quiet(out)) {
	- pcmk__set_effective_date(data_set, true, use_date);
	+ PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	+ if (pcmk__simulate_transition(data_set, cib,
	+ injections->op_fail) != transition_complete) {
	+ rc = pcmk_rc_invalid_transition;
	+ }

	- if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	- pe__set_working_set_flags(data_set, pe_flag_show_scores);
	- }
	- if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	- pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	- }
	+ if (out->is_quiet(out)) {
	+ goto simulate_done;
	+ }

	- cluster_status(data_set);
	- print_cluster_status(data_set, 0, section_opts, "Revised Cluster Status", true);
	- }
	+ set_effective_date(data_set, true, use_date);
	+
	+ if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	+ pe__set_working_set_flags(data_set, pe_flag_show_scores);
	+ }
	+ if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	+ pe__set_working_set_flags(data_set, pe_flag_show_utilization);
	}

	+ cluster_status(data_set);
	+ print_cluster_status(data_set, 0, section_opts, "Revised Cluster Status",
	+ true);
	+
	simulate_done:
	if (cib) {
	cib->cmds->signoff(cib);
	cib_delete(cib);
	}
	-
	return rc;
	}

	int
	-pcmk_simulate(xmlNodePtr xml, pe_working_set_t data_set, pcmk_injections_t *injections,
	- unsigned int flags, unsigned int section_opts, char *use_date,
	- char input_file, char graph_file, char *dot_file)
	+pcmk_simulate(xmlNodePtr xml, pe_working_set_t data_set,
	+ pcmk_injections_t *injections, unsigned int flags,
	+ unsigned int section_opts, char use_date, char input_file,
	+ char graph_file, char dot_file)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__out_prologue(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__simulate(data_set, out, injections, flags, section_opts,
	use_date, input_file, graph_file, dot_file);
	pcmk__out_epilogue(out, xml, rc);
	return rc;
	}

	void
	pcmk_free_injections(pcmk_injections_t *injections)
	{
	if (injections == NULL) {
	return;
	}

	g_list_free_full(injections->node_up, g_free);
	g_list_free_full(injections->node_down, g_free);
	g_list_free_full(injections->node_fail, g_free);
	g_list_free_full(injections->op_fail, g_free);
	g_list_free_full(injections->op_inject, g_free);
	g_list_free_full(injections->ticket_grant, g_free);
	g_list_free_full(injections->ticket_revoke, g_free);
	g_list_free_full(injections->ticket_standby, g_free);
	g_list_free_full(injections->ticket_activate, g_free);
	free(injections->quorum);
	free(injections->watchdog);

	free(injections);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index 501f61f3ba..b877beea69 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2011 +1,2011 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	resource_checks_t *
	cli_check_resource(pe_resource_t rsc, char role_s, char *managed)
	{
	pe_resource_t *parent = uber_parent(rsc);
	resource_checks_t *rc = calloc(1, sizeof(resource_checks_t));

	if (role_s) {
	enum rsc_role_e role = text2role(role_s);

	if (role == RSC_ROLE_STOPPED) {
	rc->flags \|= rsc_remain_stopped;
	} else if (pcmk_is_set(parent->flags, pe_rsc_promotable) &&
	(role == RSC_ROLE_UNPROMOTED)) {
	rc->flags \|= rsc_unpromotable;
	}
	}

	if (managed && !crm_is_true(managed)) {
	rc->flags \|= rsc_unmanaged;
	}

	if (rsc->lock_node) {
	rc->lock_node = rsc->lock_node->details->uname;
	}

	rc->rsc = rsc;
	return rc;
	}

	static GList *
	build_node_info_list(pe_resource_t *rsc)
	{
	GList *retval = NULL;

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pe_resource_t child = (pe_resource_t ) iter->data;

	for (GList *iter2 = child->running_on; iter2 != NULL; iter2 = iter2->next) {
	pe_node_t node = (pe_node_t ) iter2->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
	child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pe_resource_t rsc, const char requested_name,
	pe_working_set_t *data_set)
	{
	GList *retval = NULL;
	pe_resource_t *parent = uber_parent(rsc);

	if (pe_rsc_is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pe_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	retval = build_node_info_list(parent);

	} else if (rsc->running_on != NULL) {
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	#define XPATH_MAX 1024

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int offset = 0;
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	char *xpath_string = NULL;
	const char *xpath_base = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath_string = calloc(1, XPATH_MAX);
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s",
	xpath_base);

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);

	if (attr_set_type) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", attr_set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
	CRM_LOG_ASSERT(offset > 0);

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = ENOTUNIQ;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	done:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t out, GList/ <pe_resource_t> / ** result,
	pe_resource_t * rsc, const char * rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if(pe_clone == rsc->variant) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pe_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pe_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if(rsc->parent && pe_clone == rsc->parent->variant) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);
	} else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
	pe_resource_t *child = rsc->children->data;

	if(child->variant == pe_native) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	static bool need_init = TRUE;

	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	GList/<pe_resource_t>/ resources = NULL;
	const char *common_attr_id = attr_id;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr (out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (force == FALSE) {
	rc = find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_rc_ok && !out->is_quiet(out)) {
	out->err(out, "WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)",
	uber_parent(rsc)->id, attr_name, local_attr_id);
	out->err(out, " Delete '%s' first or use the force option to override",
	local_attr_id);
	}
	free(local_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "update", force);
	}

	/* If either attr_set or attr_id is specified,
	* one clearly intends to modify a single resource.
	* It is the last item on the resource list.*/
	for(GList *gIter = (attr_set\|\|attr_id) ? g_list_last(resources) : resources
	; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	local_attr_id = NULL;
	local_attr_set = NULL;

	rsc = (pe_resource_t*)gIter->data;
	attr_id = common_attr_id;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == pcmk_rc_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != ENXIO) {
	free(lookup_id);
	free(local_attr_id);
	g_list_free(resources);
	return rc;

	} else {
	const char *tag = crm_element_name(rsc->xml);

	if (attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_strdup_printf("%s-%s", attr_set, attr_name);
	attr_id = local_attr_id;
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(local_attr_id);
	free(local_attr_set);

	if(recursive && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	GList *lpc = NULL;

	if(need_init) {
	need_init = FALSE;
	pcmk__unpack_constraints(data_set);
	pe__clear_resource_flags_on_all(data_set, pe_rsc_allocating);
	}

	crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
	pe__set_resource_flags(rsc, pe_rsc_allocating);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if ((cons->score > 0)
	&& !pcmk_is_set(cons->dependent->flags, pe_rsc_allocating)) {
	/* Don't get into colocation loops */
	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	cli_resource_update_attribute(cons->dependent,
	cons->dependent->id, NULL,
	attr_set_type, NULL,
	attr_name, attr_value,
	recursive, cib, cib_options,
	data_set, force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pe_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	GList/<pe_resource_t>/ resources = NULL;

	if (attr_id == NULL && force == FALSE) {
	find_resource_attr(out, cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if(pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name, attr_set, attr_set_type,
	attr_id, attr_name, cib, "delete", force);
	} else {
	resources = g_list_append(resources, rsc);
	}

	for(GList *gIter = resources; gIter; gIter = gIter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *local_attr_id = NULL;

	rsc = (pe_resource_t*)gIter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);

	if (rc == ENXIO) {
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	} else if (rc != pcmk_rc_ok) {
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s", lookup_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(local_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id, pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pe_native) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pe_node_t *node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(pe_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pe_working_set_t data_set)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	const char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	interval_ms_s = crm_strdup_printf("%u",
	crm_parse_interval_spec(interval_spec));
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
	failed_id,
	pe_find_renamed\|pe_find_anon);

	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(pe_resource_t rsc, const char operation,
	const char interval_spec, pe_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	pe_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pe_working_set_t *data_set, gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	pe_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {
	GList *lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	pe_resource_t child = (pe_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, data_set,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(data_set->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pe_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pe_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->details->uname,
	rsc, operation, interval_spec,
	just_failures, data_set, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(data_set->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, data_set);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_strerror(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pe_working_set_t *data_set)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pe_node_t *node = pe_find_node(data_set->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_strerror(rc));
	return rc;
	}
	} else {
	for (GList *iter = data_set->nodes; iter; iter = iter->next) {
	pe_node_t node = (pe_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, data_set);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_strerror(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	int
	cli_resource_check(pcmk__output_t out, cib_t cib_conn, pe_resource_t *rsc)
	{
	char *role_s = NULL;
	char *managed = NULL;
	pe_resource_t *parent = uber_parent(rsc);
	int rc = pcmk_rc_no_output;
	resource_checks_t *checks = NULL;

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);

	find_resource_attr(out, cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
	NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);

	checks = cli_check_resource(rsc, role_s, managed);

	if (checks->flags != 0 \|\| checks->lock_node != NULL) {
	rc = out->message(out, "resource-check-list", checks);
	}

	free(role_s);
	free(managed);
	free(checks);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
	}

	static GHashTable *
	generate_resource_params(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, data_set);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, node, data_set);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pe_resource_t rsc, const char host)
	{
	bool found = TRUE;
	GList *hIter = NULL;
	GList *hosts = NULL;

	if(rsc == NULL) {
	return FALSE;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pe_node_t node = (pe_node_t ) hIter->data;

	if(strcmp(host, node->details->uname) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	} else if(strcmp(host, node->details->id) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if(host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = FALSE;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = FALSE;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pe_resource_t rsc = (pe_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pe_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as working set input
	*
	* This also updates the working set timestamp to the current time.
	*
	* \param[in] data_set Working set instance to update
	* \param[in] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_working_set_xml(pe_working_set_t data_set, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	data_set->input = *xml;
	data_set->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update a working set's XML input based on a CIB query
	*
	* \param[in] data_set Data set instance to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->input and data_set->now.
	*/
	static int
	update_working_set_from_cib(pcmk__output_t out, pe_working_set_t data_set,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
	return rc;
	}
	rc = update_working_set_xml(data_set, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pe_working_set_t data_set, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = data_set->priv;

	pe_reset_working_set(data_set);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = write_xml_file(data_set->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	pcmk__schedule_actions(data_set, data_set->input, NULL);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	- run_simulation(data_set, shadow_cib, NULL);
	+ pcmk__simulate_transition(data_set, shadow_cib, NULL);
	out->quiet = prev_quiet;

	rc = update_dataset(shadow_cib, data_set, FALSE);

	} else {
	cluster_status(data_set);
	}

	done:
	/* Do not free data_set->input here, we need rsc->xml to be valid later on */
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	static int
	max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc)
	{
	int delay = 0;
	int max_delay = 0;

	if(rsc && rsc->children) {
	GList *iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;

	delay = max_delay_for_resource(data_set, child);
	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
	max_delay = delay;
	}
	}

	} else if(rsc) {
	char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
	pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
	const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
	long long result_ll;

	if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
	&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
	max_delay = (int) result_ll;
	} else {
	max_delay = -1;
	}
	pe_free_action(stop);
	}

	return max_delay;
	}

	static int
	max_delay_in(pe_working_set_t * data_set, GList *resources)
	{
	int max_delay = 0;
	GList *item = NULL;

	for (item = resources; item != NULL; item = item->next) {
	int delay = 0;
	pe_resource_t rsc = pe_find_resource(data_set->resources, (const char )item->data);

	delay = max_delay_for_resource(data_set, rsc);

	if(delay > max_delay) {
	double seconds = delay / 1000.0;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
	max_delay = delay;
	}
	}

	return 5 + (max_delay / 1000);
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in] rsc The resource to restart
	* \param[in] host The host to restart the resource on (or NULL for all)
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but a two-second
	* granularity is actually used; if 0, a timeout will be
	* calculated based on the resource timeout)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pe_resource_t rsc, const char *host,
	const char move_lifetime, int timeout_ms, cib_t cib,
	int cib_options, gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = FALSE;
	char *rsc_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pe_working_set_t *data_set = NULL;

	if (!resource_is_running_on(rsc, host)) {
	const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
	if(host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	rsc_id = strdup(rsc->id);
	if ((pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) && host) {
	stop_via_ban = TRUE;
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	crm_perror(LOG_ERR, "Could not allocate working set");
	rc = ENOMEM;
	goto done;
	}

	data_set->priv = out;
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);
	rc = update_dataset(cib, data_set, FALSE);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, data_set->resources);
	current_active = get_active_resources(host, data_set->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, rsc_id, host, move_lifetime, NULL, cib,
	cib_options, promoted_role_only);

	} else {
	/* Stop the resource by setting target-role to Stopped.
	* Remember any existing target-role so we can restore it later
	* (though it only makes any difference if it's Unpromoted).
	*/
	char *lookup_id = clone_strip(rsc->id);

	find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
	NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
	free(lookup_id);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	RSC_STOPPED, FALSE, cib, cib_options,
	data_set, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	goto done;
	}

	rc = update_dataset(cib, data_set, TRUE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, data_set->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, data_set->resources);
	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, data_set, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
	cib_options, data_set, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
	goto done;
	}

	if (target_active) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, data_set, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	if (current_active) {
	g_list_free_full(current_active, free);
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	current_active = get_active_resources(NULL, data_set->resources);
	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(rsc_id, host, NULL, cib, cib_options, TRUE, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, data_set, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	XML_RSC_ATTR_TARGET_ROLE, cib, cib_options,
	data_set, force);
	}

	done:
	if (list_delta) {
	g_list_free(list_delta);
	}
	if (current_active) {
	g_list_free_full(current_active, free);
	}
	if (target_active && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	pe_free_working_set(data_set);
	return rc;
	}

	static inline bool action_is_pending(pe_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags, pe_action_optional\|pe_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pe_action_runnable)
	\|\| pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Return TRUE if any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return TRUE if any actions in the list are pending, FALSE otherwise
	*/
	static bool
	actions_are_pending(GList *actions)
	{
	GList *action;

	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t )action->data;
	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	return TRUE;
	}
	}
	return FALSE;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pe_action_t a = (pe_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s", a->id, a->uuid, a->node->details->uname);
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but one-second granularity
	* is actually used; if 0, a default will be used)
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pe_working_set_t *data_set = NULL;
	int rc = pcmk_rc_ok;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	return ENOMEM;
	}
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);

	do {

	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
	} else {
	print_pending_actions(out, data_set->actions);
	pe_free_working_set(data_set);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(data_set);
	rc = update_working_set_from_cib(out, data_set, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(data_set);
	return rc;
	}
	pcmk__schedule_actions(data_set, data_set->input, NULL);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(data_set->config_hash,
	"dc-version");

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = TRUE;
	}
	}

	} while (actions_are_pending(data_set->actions));

	pe_free_working_set(data_set);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = "validate-all";

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = "monitor";

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
	strdup(CRM_FEATURE_SET));

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	g_hash_table_replace(params, strdup(name), strdup(value));
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force,
	int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0) {
	timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, timeout_ms, params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__set_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of this standard is unsupported");
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pe_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = data_set->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if(pe_rsc_is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, data_set);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out, "Try setting target-role=Stopped first or specifying "
	"the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if(rsc->variant == pe_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (rsc->variant == pe_container \|\| pe_rsc_is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	data_set);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(pe_resource_t rsc, const char rsc_id, const char *host_name,
	const char move_lifetime, cib_t cib, int cib_options,
	pe_working_set_t *data_set, gboolean promoted_role_only,
	gboolean force)
	{
	pcmk__output_t *out = data_set->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pe_node_t *current = NULL;
	pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
	bool cur_is_dest = FALSE;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	pe_resource_t *p = uber_parent(rsc);

	if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring master option: %s is not promotable", rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
	GList *iter = NULL;
	unsigned int promoted_count = 0;
	pe_node_t *promoted_node = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	pe_resource_t child = (pe_resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == RSC_ROLE_PROMOTED) {
	rsc = child;
	promoted_node = pe__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (current && (current->details == dest->details)) {
	cur_is_dest = TRUE;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active", dest->details->uname);
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, FALSE, force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only);

	crm_trace("%s%s now prefers node %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	dest->details->uname, force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if(force && (cur_is_dest == FALSE)) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	NULL, cib, cib_options, promoted_role_only);

	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, dest->details->uname);
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 3:50 AM (4 h, 42 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018141
Default Alt Text: (163 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions