No OneTemporary
Actions

Size

206 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/schedulerd/pacemaker-schedulerd.c b/daemons/schedulerd/pacemaker-schedulerd.c
	index cee938fecd..4fc6aaa7d1 100644
	--- a/daemons/schedulerd/pacemaker-schedulerd.c
	+++ b/daemons/schedulerd/pacemaker-schedulerd.c
	@@ -1,359 +1,360 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <stdio.h>
	+#include <stdbool.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <libxml/parser.h>

	#include <crm/common/ipc_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>
	#include <crm/msg_xml.h>

	#define OPTARGS "hVc"

	static GMainLoop *mainloop = NULL;
	static qb_ipcs_service_t *ipcs = NULL;
	static pe_working_set_t *sched_data_set = NULL;

	#define get_series() was_processing_error?1:was_processing_warning?2:3

	typedef struct series_s {
	const char *name;
	const char *param;
	int wrap;
	} series_t;

	series_t series[] = {
	{"pe-unknown", "_do_not_match_anything_", -1},
	{"pe-error", "pe-error-series-max", -1},
	{"pe-warn", "pe-warn-series-max", 200},
	{"pe-input", "pe-input-series-max", 400},
	};

	void pengine_shutdown(int nsig);

	static gboolean
	process_pe_message(xmlNode msg, xmlNode xml_data, pcmk__client_t *sender)
	{
	static char *last_digest = NULL;
	static char *filename = NULL;

	const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
	const char *op = crm_element_value(msg, F_CRM_TASK);
	const char *ref = crm_element_value(msg, F_CRM_REFERENCE);

	crm_trace("Processing %s op (ref=%s)...", op, ref);

	if (op == NULL) {
	/* error */

	} else if (strcasecmp(op, CRM_OP_HELLO) == 0) {
	/* ignore */

	} else if (pcmk__str_eq(crm_element_value(msg, F_CRM_MSG_TYPE), XML_ATTR_RESPONSE, pcmk__str_casei)) {
	/* ignore */

	} else if (sys_to == NULL \|\| strcasecmp(sys_to, CRM_SYSTEM_PENGINE) != 0) {
	crm_trace("Bad sys-to %s", crm_str(sys_to));
	return FALSE;

	} else if (strcasecmp(op, CRM_OP_PECALC) == 0) {
	unsigned int seq;
	int series_id = 0;
	int series_wrap = 0;
	char *digest = NULL;
	const char *value = NULL;
	time_t execution_date = time(NULL);
	xmlNode *converted = NULL;
	xmlNode *reply = NULL;
	gboolean is_repoke = FALSE;
	gboolean process = TRUE;

	- crm_config_error = FALSE;
	- crm_config_warning = FALSE;
	+ pcmk__config_error = false;
	+ pcmk__config_warning = false;

	was_processing_error = FALSE;
	was_processing_warning = FALSE;

	if (sched_data_set == NULL) {
	sched_data_set = pe_new_working_set();
	CRM_ASSERT(sched_data_set != NULL);
	pe__set_working_set_flags(sched_data_set,
	pe_flag_no_counts\|pe_flag_no_compat);
	}

	digest = calculate_xml_versioned_digest(xml_data, FALSE, FALSE, CRM_FEATURE_SET);
	converted = copy_xml(xml_data);
	if (cli_config_update(&converted, NULL, TRUE) == FALSE) {
	sched_data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
	crm_xml_add_int(sched_data_set->graph, "transition_id", 0);
	crm_xml_add_int(sched_data_set->graph, "cluster-delay", 0);
	process = FALSE;
	free(digest);

	} else if (pcmk__str_eq(digest, last_digest, pcmk__str_casei)) {
	crm_info("Input has not changed since last time, not saving to disk");
	is_repoke = TRUE;
	free(digest);

	} else {
	free(last_digest);
	last_digest = digest;
	}

	if (process) {
	pcmk__schedule_actions(sched_data_set, converted, NULL);
	}

	series_id = get_series();
	series_wrap = series[series_id].wrap;
	value = pe_pref(sched_data_set->config_hash, series[series_id].param);

	if (value != NULL) {
	series_wrap = (int) crm_parse_ll(value, NULL);
	if (errno != 0) {
	series_wrap = series[series_id].wrap;
	}

	} else {
	pcmk__config_warn("No value specified for cluster preference: %s",
	series[series_id].param);
	}

	if (pcmk__read_series_sequence(PE_STATE_DIR, series[series_id].name,
	&seq) != pcmk_rc_ok) {
	// @TODO maybe handle errors better ...
	seq = 0;
	}
	crm_trace("Series %s: wrap=%d, seq=%u, pref=%s",
	series[series_id].name, series_wrap, seq, value);

	sched_data_set->input = NULL;
	reply = create_reply(msg, sched_data_set->graph);
	CRM_ASSERT(reply != NULL);

	if (is_repoke == FALSE) {
	free(filename);
	filename = pcmk__series_filename(PE_STATE_DIR,
	series[series_id].name, seq, true);
	}

	crm_xml_add(reply, F_CRM_TGRAPH_INPUT, filename);
	crm_xml_add_int(reply, "graph-errors", was_processing_error);
	crm_xml_add_int(reply, "graph-warnings", was_processing_warning);
	- crm_xml_add_int(reply, "config-errors", crm_config_error);
	- crm_xml_add_int(reply, "config-warnings", crm_config_warning);
	+ crm_xml_add_int(reply, "config-errors", pcmk__config_error);
	+ crm_xml_add_int(reply, "config-warnings", pcmk__config_warning);

	if (pcmk__ipc_send_xml(sender, 0, reply,
	crm_ipc_server_event) != pcmk_rc_ok) {
	int graph_file_fd = 0;
	char *graph_file = NULL;
	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	graph_file = crm_strdup_printf("%s/pengine.graph.XXXXXX",
	PE_STATE_DIR);
	graph_file_fd = mkstemp(graph_file);

	crm_err("Couldn't send transition graph to peer, writing to %s instead",
	graph_file);

	crm_xml_add(reply, F_CRM_TGRAPH, graph_file);
	write_xml_fd(sched_data_set->graph, graph_file, graph_file_fd, FALSE);

	free(graph_file);
	free_xml(first_named_child(reply, F_CRM_DATA));
	CRM_ASSERT(pcmk__ipc_send_xml(sender, 0, reply,
	crm_ipc_server_event) == pcmk_rc_ok);
	}

	free_xml(reply);
	pe_reset_working_set(sched_data_set);
	pcmk__log_transition_summary(filename);

	if (is_repoke == FALSE && series_wrap != 0) {
	unlink(filename);
	crm_xml_add_ll(xml_data, "execution-date", (long long) execution_date);
	write_xml_file(xml_data, filename, TRUE);
	pcmk__write_series_sequence(PE_STATE_DIR, series[series_id].name,
	++seq, series_wrap);
	} else {
	crm_trace("Not writing out %s: %d & %d", filename, is_repoke, series_wrap);
	}

	free_xml(converted);
	}

	return TRUE;
	}

	static int32_t
	pe_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	gboolean process_pe_message(xmlNode msg, xmlNode xml_data,
	pcmk__client_t *sender);

	static int32_t
	pe_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	pcmk__client_t *c = pcmk__find_client(qbc);
	xmlNode *msg = pcmk__client_data2xml(c, data, &id, &flags);

	pcmk__ipc_send_ack(c, id, flags, "ack");
	if (msg != NULL) {
	xmlNode *data_xml = get_message_xml(msg, F_CRM_DATA);

	process_pe_message(msg, data_xml, c);
	free_xml(msg);
	}
	return 0;
	}

	/* Error code means? */
	static int32_t
	pe_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}
	crm_trace("Connection %p", c);
	pcmk__free_client(client);
	return 0;
	}

	static void
	pe_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	pe_ipc_closed(c);
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = pe_ipc_accept,
	.connection_created = NULL,
	.msg_process = pe_ipc_dispatch,
	.connection_closed = pe_ipc_closed,
	.connection_destroyed = pe_ipc_destroy
	};

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"help", no_argument, NULL, '?',
	"\tThis text", pcmk__option_default
	},
	{
	"verbose", no_argument, NULL, 'V',
	"\tIncrease debug output", pcmk__option_default
	},
	{ 0, 0, 0, 0 }
	};

	int
	main(int argc, char **argv)
	{
	int flag;
	int index = 0;
	int argerr = 0;

	crm_log_preinit(NULL, argc, argv);
	pcmk__set_cli_options(NULL, "[options]", long_options,
	"daemon for calculating a Pacemaker cluster's "
	"response to events");

	mainloop_add_signal(SIGTERM, pengine_shutdown);

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &index, NULL);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'h': /* Help message */
	pcmk__cli_help('?', CRM_EX_OK);
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argc - optind == 1 && pcmk__str_eq("metadata", argv[optind], pcmk__str_casei)) {
	pe_metadata();
	return CRM_EX_OK;
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	pcmk__cli_help('?', CRM_EX_USAGE);
	}

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	crm_notice("Starting Pacemaker scheduler");

	if (pcmk__daemon_can_write(PE_STATE_DIR, NULL) == FALSE) {
	crm_err("Terminating due to bad permissions on " PE_STATE_DIR);
	fprintf(stderr,
	"ERROR: Bad permissions on " PE_STATE_DIR " (see logs for details)\n");
	fflush(stderr);
	return CRM_EX_FATAL;
	}

	ipcs = mainloop_add_ipc_server(CRM_SYSTEM_PENGINE, QB_IPC_SHM, &ipc_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	crm_exit(CRM_EX_FATAL);
	}

	/* Create the mainloop and run it... */
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker scheduler successfully started and accepting connections");
	g_main_loop_run(mainloop);

	pe_free_working_set(sched_data_set);
	pcmk__unregister_formats();
	crm_info("Exiting %s", crm_system_name);
	crm_exit(CRM_EX_OK);
	}

	void
	pengine_shutdown(int nsig)
	{
	mainloop_del_ipc_server(ipcs);
	pe_free_working_set(sched_data_set);
	crm_exit(CRM_EX_OK);
	}
	diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
	index 54a5ca0c29..60b3e675a1 100644
	--- a/include/crm/common/internal.h
	+++ b/include/crm/common/internal.h
	@@ -1,374 +1,388 @@
	/*
	* Copyright 2015-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_COMMON_INTERNAL__H
	#define CRM_COMMON_INTERNAL__H

	#include <unistd.h> // getpid()
	#include <stdbool.h> // bool
	#include <stdint.h> // uint8_t, uint64_t
	#include <string.h> // strcmp()
	#include <fcntl.h> // open()
	#include <sys/types.h> // uid_t, gid_t, pid_t

	#include <glib.h> // guint, GList, GHashTable
	#include <libxml/tree.h> // xmlNode

	#include <crm/common/util.h> // crm_strdup_printf()
	#include <crm/common/logging.h> // do_crm_log_unlikely(), etc.
	#include <crm/common/mainloop.h> // mainloop_io_t, struct ipc_client_callbacks
	#include <crm/common/strings_internal.h>

	// Internal ACL-related utilities (from acl.c)

	char *pcmk__uid2username(uid_t uid);
	const char pcmk__update_acl_user(xmlNode request, const char *field,
	const char *peer_user);

	#if ENABLE_ACL
	# include <string.h>
	static inline bool
	pcmk__is_privileged(const char *user)
	{
	return user && (!strcmp(user, CRM_DAEMON_USER) \|\| !strcmp(user, "root"));
	}
	#endif


	#if SUPPORT_CIBSECRETS
	// Internal CIB utilities (from cib_secrets.c) */

	int pcmk__substitute_secrets(const char rsc_id, GHashTable params);
	#endif


	/* internal digest-related utilities (from digest.c) */

	bool pcmk__verify_digest(xmlNode input, const char expected);


	/* internal I/O utilities (from io.c) */

	int pcmk__real_path(const char path, char *resolved_path);

	char pcmk__series_filename(const char directory, const char *series,
	int sequence, bool bzip);
	int pcmk__read_series_sequence(const char directory, const char series,
	unsigned int *seq);
	void pcmk__write_series_sequence(const char directory, const char series,
	unsigned int sequence, int max);
	int pcmk__chown_series_sequence(const char directory, const char series,
	uid_t uid, gid_t gid);

	int pcmk__build_path(const char *path_c, mode_t mode);
	bool pcmk__daemon_can_write(const char dir, const char file);
	void pcmk__sync_directory(const char *name);

	int pcmk__file_contents(const char filename, char *contents);
	int pcmk__write_sync(int fd, const char *contents);
	int pcmk__set_nonblocking(int fd);
	const char *pcmk__get_tmpdir(void);

	void pcmk__close_fds_in_child(bool);

	/*!
	* \internal
	* \brief Open /dev/null to consume next available file descriptor
	*
	* Open /dev/null, disregarding the result. This is intended when daemonizing to
	* be able to null stdin, stdout, and stderr.
	*
	* \param[in] flags O_RDONLY (stdin) or O_WRONLY (stdout and stderr)
	*/
	static inline void
	pcmk__open_devnull(int flags)
	{
	// Static analysis clutter
	// cppcheck-suppress leakReturnValNotUsed
	(void) open("/dev/null", flags);
	}


	/* internal logging utilities */

	+/*!
	+ * \internal
	+ * \brief Log a configuration error
	+ *
	+ * \param[in] fmt printf(3)-style format string
	+ * \param[in] ... Arguments for format string
	+ */
	# define pcmk__config_err(fmt...) do { \
	- crm_config_error = TRUE; \
	+ pcmk__config_error = true; \
	crm_err(fmt); \
	} while (0)

	+/*!
	+ * \internal
	+ * \brief Log a configuration warning
	+ *
	+ * \param[in] fmt printf(3)-style format string
	+ * \param[in] ... Arguments for format string
	+ */
	# define pcmk__config_warn(fmt...) do { \
	- crm_config_warning = TRUE; \
	+ pcmk__config_warning = true; \
	crm_warn(fmt); \
	} while (0)

	/*!
	* \internal
	* \brief Execute code depending on whether message would be logged
	*
	* This is similar to do_crm_log_unlikely() except instead of logging, it either
	* continues past this statement or executes else_action depending on whether a
	* message of the given severity would be logged or not. This allows whole
	* blocks of code to be skipped if tracing or debugging is turned off.
	*
	* \param[in] level Severity at which to continue past this statement
	* \param[in] else_action Code block to execute if severity would not be logged
	*
	* \note else_action must not contain a break or continue statement
	*/
	# define pcmk__log_else(level, else_action) do { \
	static struct qb_log_callsite *trace_cs = NULL; \
	\
	if (trace_cs == NULL) { \
	trace_cs = qb_log_callsite_get(__func__, __FILE__, "log_else", \
	level, __LINE__, 0); \
	} \
	if (!crm_is_callsite_active(trace_cs, level, 0)) { \
	else_action; \
	} \
	} while(0)


	/* internal main loop utilities (from mainloop.c) */

	int pcmk__add_mainloop_ipc(crm_ipc_t ipc, int priority, void userdata,
	struct ipc_client_callbacks *callbacks,
	mainloop_io_t **source);


	/* internal procfs utilities (from procfs.c) */

	pid_t pcmk__procfs_pid_of(const char *name);
	unsigned int pcmk__procfs_num_cores(void);


	/* internal XML schema functions (from xml.c) */

	void crm_schema_init(void);
	void crm_schema_cleanup(void);


	/* internal functions related to process IDs (from pid.c) */

	/*!
	* \internal
	* \brief Check whether process exists (by PID and optionally executable path)
	*
	* \param[in] pid PID of process to check
	* \param[in] daemon If not NULL, path component to match with procfs entry
	*
	* \return Standard Pacemaker return code
	* \note Particular return codes of interest include pcmk_rc_ok for alive,
	* ESRCH for process is not alive (verified by kill and/or executable path
	* match), EACCES for caller unable or not allowed to check. A result of
	* "alive" is less reliable when \p daemon is not provided or procfs is
	* not available, since there is no guarantee that the PID has not been
	* recycled for another process.
	* \note This function cannot be used to verify \e authenticity of the process.
	*/
	int pcmk__pid_active(pid_t pid, const char *daemon);

	int pcmk__read_pidfile(const char filename, pid_t pid);
	int pcmk__pidfile_matches(const char *filename, pid_t expected_pid,
	const char expected_name, pid_t pid);
	int pcmk__lock_pidfile(const char filename, const char name);


	/* internal functions related to resource operations (from operations.c) */

	// printf-style format to create operation ID from resource, action, interval
	#define PCMK__OP_FMT "%s_%s_%u"

	char pcmk__op_key(const char rsc_id, const char *op_type, guint interval_ms);
	char pcmk__notify_key(const char rsc_id, const char *notify_type,
	const char *op_type);
	char *pcmk__transition_key(int transition_id, int action_id, int target_rc,
	const char *node);
	void pcmk__filter_op_for_digest(xmlNode *param_set);


	// bitwise arithmetic utilities

	/*!
	* \internal
	* \brief Set specified flags in a flag group
	*
	* \param[in] function Function name of caller
	* \param[in] line Line number of caller
	* \param[in] log_level Log a message at this level
	* \param[in] flag_type Label describing this flag group (for logging)
	* \param[in] target Name of object whose flags these are (for logging)
	* \param[in] flag_group Flag group being manipulated
	* \param[in] flags Which flags in the group should be set
	* \param[in] flags_str Readable equivalent of \p flags (for logging)
	*
	* \return Possibly modified flag group
	*/
	static inline uint64_t
	pcmk__set_flags_as(const char *function, int line, uint8_t log_level,
	const char flag_type, const char target,
	uint64_t flag_group, uint64_t flags, const char *flags_str)
	{
	uint64_t result = flag_group \| flags;

	if (result != flag_group) {
	do_crm_log_unlikely(log_level,
	"%s flags 0x%.8llx (%s) for %s set by %s:%d",
	((flag_type == NULL)? "Group of" : flag_type),
	(unsigned long long) flags,
	((flags_str == NULL)? "flags" : flags_str),
	((target == NULL)? "target" : target),
	function, line);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Clear specified flags in a flag group
	*
	* \param[in] function Function name of caller
	* \param[in] line Line number of caller
	* \param[in] log_level Log a message at this level
	* \param[in] flag_type Label describing this flag group (for logging)
	* \param[in] target Name of object whose flags these are (for logging)
	* \param[in] flag_group Flag group being manipulated
	* \param[in] flags Which flags in the group should be cleared
	* \param[in] flags_str Readable equivalent of \p flags (for logging)
	*
	* \return Possibly modified flag group
	*/
	static inline uint64_t
	pcmk__clear_flags_as(const char *function, int line, uint8_t log_level,
	const char flag_type, const char target,
	uint64_t flag_group, uint64_t flags, const char *flags_str)
	{
	uint64_t result = flag_group & ~flags;

	if (result != flag_group) {
	do_crm_log_unlikely(log_level,
	"%s flags 0x%.8llx (%s) for %s cleared by %s:%d",
	((flag_type == NULL)? "Group of" : flag_type),
	(unsigned long long) flags,
	((flags_str == NULL)? "flags" : flags_str),
	((target == NULL)? "target" : target),
	function, line);
	}
	return result;
	}

	// miscellaneous utilities (from utils.c)

	const char pcmk_message_name(const char name);

	extern int pcmk__score_red;
	extern int pcmk__score_green;
	extern int pcmk__score_yellow;

	/*!
	* \internal
	* \brief Resize a dynamically allocated memory block
	*
	* \param[in] ptr Memory block to resize (or NULL to allocate new memory)
	* \param[in] size New size of memory block in bytes (must be > 0)
	*
	* \return Pointer to resized memory block
	*
	* \note This asserts on error, so the result is guaranteed to be non-NULL
	* (which is the main advantage of this over directly using realloc()).
	*/
	static inline void *
	pcmk__realloc(void *ptr, size_t size)
	{
	void *new_ptr;

	// realloc(p, 0) can replace free(p) but this wrapper can't
	CRM_ASSERT(size > 0);

	new_ptr = realloc(ptr, size);
	if (new_ptr == NULL) {
	free(ptr);
	abort();
	}
	return new_ptr;
	}


	/* Error domains for use with g_set_error (from results.c) */

	GQuark pcmk__rc_error_quark(void);
	GQuark pcmk__exitc_error_quark(void);

	#define PCMK__RC_ERROR pcmk__rc_error_quark()
	#define PCMK__EXITC_ERROR pcmk__exitc_error_quark()

	static inline char *
	pcmk__getpid_s(void)
	{
	return crm_strdup_printf("%lu", (unsigned long) getpid());
	}

	// More efficient than g_list_length(list) == 1
	static inline bool
	pcmk__list_of_1(GList *list)
	{
	return list && (list->next == NULL);
	}

	// More efficient than g_list_length(list) > 1
	static inline bool
	pcmk__list_of_multiple(GList *list)
	{
	return list && (list->next != NULL);
	}

	/* convenience functions for failure-related node attributes */

	#define PCMK__FAIL_COUNT_PREFIX "fail-count"
	#define PCMK__LAST_FAILURE_PREFIX "last-failure"

	/*!
	* \internal
	* \brief Generate a failure-related node attribute name for a resource
	*
	* \param[in] prefix Start of attribute name
	* \param[in] rsc_id Resource name
	* \param[in] op Operation name
	* \param[in] interval_ms Operation interval
	*
	* \return Newly allocated string with attribute name
	*
	* \note Failure attributes are named like PREFIX-RSC#OP_INTERVAL (for example,
	* "fail-count-myrsc#monitor_30000"). The '#' is used because it is not
	* a valid character in a resource ID, to reliably distinguish where the
	* operation name begins. The '_' is used simply to be more comparable to
	* action labels like "myrsc_monitor_30000".
	*/
	static inline char *
	pcmk__fail_attr_name(const char prefix, const char rsc_id, const char *op,
	guint interval_ms)
	{
	CRM_CHECK(prefix && rsc_id && op, return NULL);
	return crm_strdup_printf("%s-%s#%s_%u", prefix, rsc_id, op, interval_ms);
	}

	static inline char *
	pcmk__failcount_name(const char rsc_id, const char op, guint interval_ms)
	{
	return pcmk__fail_attr_name(PCMK__FAIL_COUNT_PREFIX, rsc_id, op,
	interval_ms);
	}

	static inline char *
	pcmk__lastfailure_name(const char rsc_id, const char op, guint interval_ms)
	{
	return pcmk__fail_attr_name(PCMK__LAST_FAILURE_PREFIX, rsc_id, op,
	interval_ms);
	}

	#endif /* CRM_COMMON_INTERNAL__H */
	diff --git a/include/crm/common/logging.h b/include/crm/common/logging.h
	index 86bb108a2a..1aa8c21f1c 100644
	--- a/include/crm/common/logging.h
	+++ b/include/crm/common/logging.h
	@@ -1,398 +1,403 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Wrappers for and extensions to libqb logging
	* \ingroup core
	*/

	#ifndef CRM_LOGGING__H
	# define CRM_LOGGING__H

	# include <stdio.h>
	# include <glib.h>
	# include <qb/qblog.h>
	# include <libxml/tree.h>

	/* Define custom log priorities.
	*
	* syslog(3) uses int for priorities, but libqb's struct qb_log_callsite uses
	* uint8_t, so make sure they fit in the latter.
	*/

	// Define something even less desired than debug
	# ifndef LOG_TRACE
	# define LOG_TRACE (LOG_DEBUG+1)
	# endif

	// Print message to stdout instead of logging it
	# ifndef LOG_STDOUT
	# define LOG_STDOUT 254
	# endif

	// Don't send message anywhere
	# ifndef LOG_NEVER
	# define LOG_NEVER 255
	# endif

	/* "Extended information" logging support */
	#ifdef QB_XS
	# define CRM_XS QB_XS
	# define crm_extended_logging(t, e) qb_log_ctl((t), QB_LOG_CONF_EXTENDED, (e))
	#else
	# define CRM_XS "\|"

	/* A caller might want to check the return value, so we can't define this as a
	* no-op, and we can't simply define it to be 0 because gcc will then complain
	* when the value isn't checked.
	*/
	static inline int
	crm_extended_logging(int t, int e)
	{
	return 0;
	}
	#endif

	extern unsigned int crm_log_level;
	-extern gboolean crm_config_error;
	-extern gboolean crm_config_warning;
	extern unsigned int crm_trace_nonlog;

	+/* These are set when a configuration issue is found, and turn on extra messages
	+ * at the end of processing. They are set via wrapper functions and do not need
	+ * to be set directly.
	+ */
	+extern bool pcmk__config_error;
	+extern bool pcmk__config_warning;
	+
	enum xml_log_options
	{
	xml_log_option_filtered = 0x0001,
	xml_log_option_formatted = 0x0002,
	xml_log_option_text = 0x0004, /* add this option to dump text into xml */
	xml_log_option_full_fledged = 0x0008, /* crm_xml_dump: serialize using libxml */
	xml_log_option_diff_plus = 0x0010,
	xml_log_option_diff_minus = 0x0020,
	xml_log_option_diff_short = 0x0040,
	xml_log_option_diff_all = 0x0100,
	xml_log_option_dirty_add = 0x1000,
	xml_log_option_open = 0x2000,
	xml_log_option_children = 0x4000,
	xml_log_option_close = 0x8000,
	};

	void crm_enable_blackbox(int nsig);
	void crm_disable_blackbox(int nsig);
	void crm_write_blackbox(int nsig, struct qb_log_callsite *callsite);

	void crm_update_callsites(void);

	void crm_log_deinit(void);

	gboolean crm_log_cli_init(const char *entity);

	void crm_log_preinit(const char entity, int argc, char *argv);
	gboolean crm_log_init(const char *entity, uint8_t level, gboolean daemon,
	gboolean to_stderr, int argc, char **argv, gboolean quiet);

	void crm_log_args(int argc, char **argv);
	void crm_log_output_fn(const char file, const char function, int line, int level,
	const char prefix, const char output);

	// Log a block of text line by line
	# define crm_log_output(level, prefix, output) crm_log_output_fn(__FILE__, __FUNCTION__, __LINE__, level, prefix, output)

	gboolean crm_add_logfile(const char *filename);

	void crm_bump_log_level(int argc, char **argv);

	void crm_enable_stderr(int enable);

	gboolean crm_is_callsite_active(struct qb_log_callsite *cs, uint8_t level, uint32_t tags);

	void log_data_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, gboolean formatted);

	/* returns the old value */
	unsigned int set_crm_log_level(unsigned int level);

	unsigned int get_crm_log_level(void);

	/*
	* Throughout the macros below, note the leading, pre-comma, space in the
	* various ' , ##args' occurrences to aid portability across versions of 'gcc'.
	* https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html#Variadic-Macros
	*/
	#if defined(__clang__)
	# define CRM_TRACE_INIT_DATA(name)
	# else
	# include <assert.h> // required by QB_LOG_INIT_DATA() macro
	# define CRM_TRACE_INIT_DATA(name) QB_LOG_INIT_DATA(name)
	#endif

	/* Using "switch" instead of "if" in these macro definitions keeps
	* static analysis from complaining about constant evaluations
	*/

	/*!
	* \brief Log a message
	*
	* \param[in] level Priority at which to log the message
	* \param[in] fmt printf-style format string literal for message
	* \param[in] args Any arguments needed by format string
	*
	* \note This is a macro, and \p level may be evaluated more than once.
	*/
	# define do_crm_log(level, fmt, args...) do { \
	switch (level) { \
	case LOG_STDOUT: \
	printf(fmt "\n" , ##args); \
	break; \
	case LOG_NEVER: \
	break; \
	default: \
	qb_log_from_external_source(__func__, __FILE__, fmt, \
	(level), __LINE__, 0 , ##args); \
	break; \
	} \
	} while (0)

	/*!
	* \brief Log a message that is likely to be filtered out
	*
	* \param[in] level Priority at which to log the message
	* \param[in] fmt printf-style format string for message
	* \param[in] args Any arguments needed by format string
	*
	* \note This is a macro, and \p level may be evaluated more than once.
	* This does nothing when level is LOG_STDOUT.
	*/
	# define do_crm_log_unlikely(level, fmt, args...) do { \
	switch (level) { \
	case LOG_STDOUT: case LOG_NEVER: \
	break; \
	default: { \
	static struct qb_log_callsite *trace_cs = NULL; \
	if (trace_cs == NULL) { \
	trace_cs = qb_log_callsite_get(__func__, __FILE__, fmt, \
	(level), __LINE__, 0); \
	} \
	if (crm_is_callsite_active(trace_cs, (level), 0)) { \
	qb_log_from_external_source(__func__, __FILE__, fmt, \
	(level), __LINE__, 0 , ##args); \
	} \
	} \
	break; \
	} \
	} while (0)

	# define CRM_LOG_ASSERT(expr) do { \
	if(__unlikely((expr) == FALSE)) { \
	static struct qb_log_callsite *core_cs = NULL; \
	if(core_cs == NULL) { \
	core_cs = qb_log_callsite_get(__func__, __FILE__, "log-assert", LOG_TRACE, __LINE__, 0); \
	} \
	crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr, \
	core_cs?core_cs->targets:FALSE, TRUE); \
	} \
	} while(0)

	/* 'failure_action' MUST NOT be 'continue' as it will apply to the
	* macro's do-while loop
	*/
	# define CRM_CHECK(expr, failure_action) do { \
	if(__unlikely((expr) == FALSE)) { \
	static struct qb_log_callsite *core_cs = NULL; \
	if(core_cs == NULL) { \
	core_cs = qb_log_callsite_get(__func__, __FILE__, "check-assert", LOG_TRACE, __LINE__, 0); \
	} \
	crm_abort(__FILE__, __FUNCTION__, __LINE__, #expr, \
	core_cs?core_cs->targets:FALSE, TRUE); \
	failure_action; \
	} \
	} while(0)

	/*!
	* \brief Log XML line-by-line in a formatted fashion
	*
	* \param[in] level Priority at which to log the messages
	* \param[in] text Prefix for each line
	* \param[in] xml XML to log
	*
	* \note This is a macro, and \p level may be evaluated more than once.
	* This does nothing when level is LOG_STDOUT.
	*/
	# define do_crm_log_xml(level, text, xml) do { \
	switch (level) { \
	case LOG_STDOUT: case LOG_NEVER: \
	break; \
	default: { \
	static struct qb_log_callsite *xml_cs = NULL; \
	if (xml_cs == NULL) { \
	xml_cs = qb_log_callsite_get(__func__, __FILE__, \
	"xml-blob", (level), __LINE__, 0); \
	} \
	if (crm_is_callsite_active(xml_cs, (level), 0)) { \
	log_data_element((level), __FILE__, __FUNCTION__, \
	__LINE__, text, xml, 1, xml_log_option_formatted); \
	} \
	} \
	break; \
	} \
	} while(0)

	/*!
	* \brief Log a message as if it came from a different code location
	*
	* \param[in] level Priority at which to log the message
	* \param[in] file Source file name to use instead of __FILE__
	* \param[in] function Source function name to use instead of __func__
	* \param[in] line Source line number to use instead of __line__
	* \param[in] fmt printf-style format string literal for message
	* \param[in] args Any arguments needed by format string
	*
	* \note This is a macro, and \p level may be evaluated more than once.
	*/
	# define do_crm_log_alias(level, file, function, line, fmt, args...) do { \
	switch (level) { \
	case LOG_STDOUT: \
	printf(fmt "\n" , ##args); \
	break; \
	case LOG_NEVER: \
	break; \
	default: \
	qb_log_from_external_source(function, file, fmt, (level), \
	line, 0 , ##args); \
	break; \
	} \
	} while (0)

	/*!
	* \brief Send a system error message to both the log and stderr
	*
	* \param[in] level Priority at which to log the message
	* \param[in] fmt printf-style format string for message
	* \param[in] args Any arguments needed by format string
	*
	* \deprecated One of the other logging functions should be used with
	* pcmk_strerror() instead.
	* \note This is a macro, and \p level may be evaluated more than once.
	* \note Because crm_perror() adds the system error message and error number
	* onto the end of fmt, that information will become extended information
	* if CRM_XS is used inside fmt and will not show up in syslog.
	*/
	# define crm_perror(level, fmt, args...) do { \
	switch (level) { \
	case LOG_NEVER: \
	break; \
	default: { \
	const char *err = strerror(errno); \
	/* cast to int makes coverity happy when level == 0 */ \
	if ((level) <= (int) crm_log_level) { \
	fprintf(stderr, fmt ": %s (%d)\n" , ##args, err, errno);\
	} \
	do_crm_log((level), fmt ": %s (%d)" , ##args, err, errno); \
	} \
	break; \
	} \
	} while (0)

	/*!
	* \brief Log a message with a tag (for use with PCMK_trace_tags)
	*
	* \param[in] level Priority at which to log the message
	* \param[in] tag String to tag message with
	* \param[in] fmt printf-style format string for message
	* \param[in] args Any arguments needed by format string
	*
	* \note This is a macro, and \p level may be evaluated more than once.
	* This does nothing when level is LOG_STDOUT.
	*/
	# define crm_log_tag(level, tag, fmt, args...) do { \
	switch (level) { \
	case LOG_STDOUT: case LOG_NEVER: \
	break; \
	default: { \
	static struct qb_log_callsite *trace_tag_cs = NULL; \
	int converted_tag = g_quark_try_string(tag); \
	if (trace_tag_cs == NULL) { \
	trace_tag_cs = qb_log_callsite_get(__func__, __FILE__, \
	fmt, (level), __LINE__, converted_tag); \
	} \
	if (crm_is_callsite_active(trace_tag_cs, (level), \
	converted_tag)) { \
	qb_log_from_external_source(__func__, __FILE__, fmt, \
	(level), __LINE__, converted_tag , ##args); \
	} \
	} \
	} \
	} while (0)

	# define crm_emerg(fmt, args...) qb_log(LOG_EMERG, fmt , ##args)
	# define crm_crit(fmt, args...) qb_logt(LOG_CRIT, 0, fmt , ##args)
	# define crm_err(fmt, args...) qb_logt(LOG_ERR, 0, fmt , ##args)
	# define crm_warn(fmt, args...) qb_logt(LOG_WARNING, 0, fmt , ##args)
	# define crm_notice(fmt, args...) qb_logt(LOG_NOTICE, 0, fmt , ##args)
	# define crm_info(fmt, args...) qb_logt(LOG_INFO, 0, fmt , ##args)

	# define crm_debug(fmt, args...) do_crm_log_unlikely(LOG_DEBUG, fmt , ##args)
	# define crm_trace(fmt, args...) do_crm_log_unlikely(LOG_TRACE, fmt , ##args)

	# define crm_log_xml_crit(xml, text) do_crm_log_xml(LOG_CRIT, text, xml)
	# define crm_log_xml_err(xml, text) do_crm_log_xml(LOG_ERR, text, xml)
	# define crm_log_xml_warn(xml, text) do_crm_log_xml(LOG_WARNING, text, xml)
	# define crm_log_xml_notice(xml, text) do_crm_log_xml(LOG_NOTICE, text, xml)
	# define crm_log_xml_info(xml, text) do_crm_log_xml(LOG_INFO, text, xml)
	# define crm_log_xml_debug(xml, text) do_crm_log_xml(LOG_DEBUG, text, xml)
	# define crm_log_xml_trace(xml, text) do_crm_log_xml(LOG_TRACE, text, xml)

	# define crm_log_xml_explicit(xml, text) do { \
	static struct qb_log_callsite *digest_cs = NULL; \
	digest_cs = qb_log_callsite_get( \
	__func__, __FILE__, text, LOG_TRACE, __LINE__, \
	crm_trace_nonlog); \
	if (digest_cs && digest_cs->targets) { \
	do_crm_log_xml(LOG_TRACE, text, xml); \
	} \
	} while(0)

	# define crm_str(x) (const char*)(x?x:"<null>")

	#ifndef PCMK__NO_COMPAT

	/* Everything here is deprecated and kept only for public API backward
	* compatibility. It will be moved to compatibility.h in a future release.
	*/

	/*!
	* \brief Log a message using constant priority
	*
	* \param[in] level Priority at which to log the message
	* \param[in] fmt printf-style format string literal for message
	* \param[in] args Any arguments needed by format string
	*
	* \deprecated Use one of the other logging functions instead
	* \note This is a macro, and \p level may be evaluated more than once.
	* This does nothing when level is LOG_STDOUT.
	*/
	# define do_crm_log_always(level, fmt, args...) do { \
	switch (level) { \
	case LOG_STDOUT: case LOG_NEVER: \
	break; \
	default: \
	qb_log((level), fmt , ##args); \
	break; \
	} \
	} while (0)

	#endif // PCMK__NO_COMPAT

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
	index c2acfa796c..c3a69f0f35 100644
	--- a/include/crm/pengine/internal.h
	+++ b/include/crm/pengine/internal.h
	@@ -1,596 +1,604 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PE_INTERNAL__H
	# define PE_INTERNAL__H
	# include <string.h>
	# include <crm/pengine/status.h>
	# include <crm/pengine/remote_internal.h>
	# include <crm/common/internal.h>
	# include <crm/common/output_internal.h>

	# define pe_rsc_info(rsc, fmt, args...) crm_log_tag(LOG_INFO, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
	# define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)

	-# define pe_err(fmt...) { was_processing_error = TRUE; crm_config_error = TRUE; crm_err(fmt); }
	-# define pe_warn(fmt...) { was_processing_warning = TRUE; crm_config_warning = TRUE; crm_warn(fmt); }
	+# define pe_err(fmt...) do { \
	+ was_processing_error = TRUE; \
	+ pcmk__config_err(fmt); \
	+ } while (0)
	+
	+# define pe_warn(fmt...) do { \
	+ was_processing_warning = TRUE; \
	+ pcmk__config_warn(fmt); \
	+ } while (0)
	+
	# define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
	# define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }

	#define pe__set_working_set_flags(working_set, flags_to_set) do { \
	(working_set)->flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_working_set_flags(working_set, flags_to_clear) do { \
	(working_set)->flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Working set", crm_system_name, \
	(working_set)->flags, (flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_resource_flags(resource, flags_to_set) do { \
	(resource)->flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_resource_flags(resource, flags_to_clear) do { \
	(resource)->flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Resource", (resource)->id, (resource)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	#define pe__set_action_flags(action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags(action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_raw_action_flags(action_flags, action_name, flags_to_set) do { \
	action_flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Action", action_name, \
	(action_flags), \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_raw_action_flags(action_flags, action_name, flags_to_clear) do { \
	action_flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, \
	"Action", action_name, \
	(action_flags), \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_action_flags_as(function, line, action, flags_to_set) do { \
	(action)->flags = pcmk__set_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_action_flags_as(function, line, action, flags_to_clear) do { \
	(action)->flags = pcmk__clear_flags_as((function), (line), \
	LOG_TRACE, \
	"Action", (action)->uuid, \
	(action)->flags, \
	(flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_order_flags(order_flags, flags_to_set) do { \
	order_flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define pe__clear_order_flags(order_flags, flags_to_clear) do { \
	order_flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, LOG_TRACE, \
	"Ordering", "constraint", \
	order_flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	#define pe__set_graph_flags(graph_flags, gr_action, flags_to_set) do { \
	graph_flags = pcmk__set_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pe__clear_graph_flags(graph_flags, gr_action, flags_to_clear) do { \
	graph_flags = pcmk__clear_flags_as(__FUNCTION__, __LINE__, \
	LOG_TRACE, "Graph", \
	(gr_action)->uuid, graph_flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	// Some warnings we don't want to print every transition

	enum pe_warn_once_e {
	pe_wo_blind = 0x0001,
	pe_wo_restart_type = 0x0002,
	pe_wo_role_after = 0x0004,
	pe_wo_poweroff = 0x0008,
	pe_wo_require_all = 0x0010,
	pe_wo_order_score = 0x0020,
	pe_wo_neg_threshold = 0x0040,
	};

	extern uint32_t pe_wo;

	#define pe_warn_once(pe_wo_bit, fmt...) do { \
	if (!pcmk_is_set(pe_wo, pe_wo_bit)) { \
	if (pe_wo_bit == pe_wo_blind) { \
	crm_warn(fmt); \
	} else { \
	pe_warn(fmt); \
	} \
	pe_wo = pcmk__set_flags_as(__FUNCTION__, __LINE__, LOG_TRACE, \
	"Warn-once", "logging", pe_wo, \
	(pe_wo_bit), #pe_wo_bit); \
	} \
	} while (0);


	typedef struct pe__location_constraint_s {
	char *id; // Constraint XML ID
	pe_resource_t *rsc_lh; // Resource being located
	enum rsc_role_e role_filter; // Role to locate
	enum pe_discover_e discover_mode; // Resource discovery
	GListPtr node_list_rh; // List of pe_node_t*
	} pe__location_t;

	typedef struct pe__order_constraint_s {
	int id;
	enum pe_ordering type;

	void *lh_opaque;
	pe_resource_t *lh_rsc;
	pe_action_t *lh_action;
	char *lh_action_task;

	void *rh_opaque;
	pe_resource_t *rh_rsc;
	pe_action_t *rh_action;
	char *rh_action_task;
	} pe__ordering_t;

	typedef struct notify_data_s {
	GSList *keys; // Environment variable name/value pairs

	const char *action;

	pe_action_t *pre;
	pe_action_t *post;
	pe_action_t *pre_done;
	pe_action_t *post_done;

	GListPtr active; /* notify_entry_t* */
	GListPtr inactive; /* notify_entry_t* */
	GListPtr start; /* notify_entry_t* */
	GListPtr stop; /* notify_entry_t* */
	GListPtr demote; /* notify_entry_t* */
	GListPtr promote; /* notify_entry_t* */
	GListPtr master; /* notify_entry_t* */
	GListPtr slave; /* notify_entry_t* */
	GHashTable *allowed_nodes;

	} notify_data_t;

	bool pe_can_fence(pe_working_set_t data_set, pe_node_t node);

	int pe__add_scores(int score1, int score2);
	void add_hash_param(GHashTable * hash, const char name, const char value);

	char native_parameter(pe_resource_t rsc, pe_node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set);
	pe_node_t native_location(const pe_resource_t rsc, GList **list, int current);

	void pe_metadata(void);
	void verify_pe_options(GHashTable * options);

	void common_update_score(pe_resource_t * rsc, const char *id, int score);
	void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set);

	gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
	gboolean pe__unpack_bundle(pe_resource_t rsc, pe_working_set_t data_set);

	pe_resource_t native_find_rsc(pe_resource_t rsc, const char id, const pe_node_t node,
	int flags);

	gboolean native_active(pe_resource_t * rsc, gboolean all);
	gboolean group_active(pe_resource_t * rsc, gboolean all);
	gboolean clone_active(pe_resource_t * rsc, gboolean all);
	gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);

	void native_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void group_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void clone_print(pe_resource_t * rsc, const char pre_text, long options, void print_data);
	void pe__print_bundle(pe_resource_t rsc, const char pre_text, long options,
	void *print_data);

	int pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...);
	char pe__node_display_name(pe_node_t node, bool print_detail);

	static inline const char *
	pe__rsc_bool_str(pe_resource_t *rsc, uint64_t rsc_flag)
	{
	return pcmk__btoa(pcmk_is_set(rsc->flags, rsc_flag));
	}

	int pe__ban_html(pcmk__output_t *out, va_list args);
	int pe__ban_text(pcmk__output_t *out, va_list args);
	int pe__ban_xml(pcmk__output_t *out, va_list args);
	int pe__clone_xml(pcmk__output_t *out, va_list args);
	int pe__clone_html(pcmk__output_t *out, va_list args);
	int pe__clone_text(pcmk__output_t *out, va_list args);
	int pe__cluster_counts_html(pcmk__output_t *out, va_list args);
	int pe__cluster_counts_text(pcmk__output_t *out, va_list args);
	int pe__cluster_counts_xml(pcmk__output_t *out, va_list args);
	int pe__cluster_dc_html(pcmk__output_t *out, va_list args);
	int pe__cluster_dc_text(pcmk__output_t *out, va_list args);
	int pe__cluster_dc_xml(pcmk__output_t *out, va_list args);
	int pe__cluster_maint_mode_html(pcmk__output_t *out, va_list args);
	int pe__cluster_maint_mode_text(pcmk__output_t *out, va_list args);
	int pe__cluster_options_html(pcmk__output_t *out, va_list args);
	int pe__cluster_options_log(pcmk__output_t *out, va_list args);
	int pe__cluster_options_text(pcmk__output_t *out, va_list args);
	int pe__cluster_options_xml(pcmk__output_t *out, va_list args);
	int pe__cluster_stack_html(pcmk__output_t *out, va_list args);
	int pe__cluster_stack_text(pcmk__output_t *out, va_list args);
	int pe__cluster_stack_xml(pcmk__output_t *out, va_list args);
	int pe__cluster_summary(pcmk__output_t *out, va_list args);
	int pe__cluster_summary_html(pcmk__output_t *out, va_list args);
	int pe__cluster_times_html(pcmk__output_t *out, va_list args);
	int pe__cluster_times_xml(pcmk__output_t *out, va_list args);
	int pe__cluster_times_text(pcmk__output_t *out, va_list args);
	int pe__failed_action_text(pcmk__output_t *out, va_list args);
	int pe__failed_action_xml(pcmk__output_t *out, va_list args);
	int pe__group_xml(pcmk__output_t *out, va_list args);
	int pe__group_html(pcmk__output_t *out, va_list args);
	int pe__group_text(pcmk__output_t *out, va_list args);
	int pe__bundle_xml(pcmk__output_t *out, va_list args);
	int pe__bundle_html(pcmk__output_t *out, va_list args);
	int pe__bundle_text(pcmk__output_t *out, va_list args);
	int pe__node_html(pcmk__output_t *out, va_list args);
	int pe__node_text(pcmk__output_t *out, va_list args);
	int pe__node_xml(pcmk__output_t *out, va_list args);
	int pe__node_attribute_html(pcmk__output_t *out, va_list args);
	int pe__node_attribute_text(pcmk__output_t *out, va_list args);
	int pe__node_attribute_xml(pcmk__output_t *out, va_list args);
	int pe__node_list_html(pcmk__output_t *out, va_list args);
	int pe__node_list_text(pcmk__output_t *out, va_list args);
	int pe__node_list_xml(pcmk__output_t *out, va_list args);
	int pe__op_history_text(pcmk__output_t *out, va_list args);
	int pe__op_history_xml(pcmk__output_t *out, va_list args);
	int pe__resource_history_text(pcmk__output_t *out, va_list args);
	int pe__resource_history_xml(pcmk__output_t *out, va_list args);
	int pe__resource_xml(pcmk__output_t *out, va_list args);
	int pe__resource_html(pcmk__output_t *out, va_list args);
	int pe__resource_text(pcmk__output_t *out, va_list args);
	int pe__ticket_html(pcmk__output_t *out, va_list args);
	int pe__ticket_text(pcmk__output_t *out, va_list args);
	int pe__ticket_xml(pcmk__output_t *out, va_list args);

	void native_free(pe_resource_t * rsc);
	void group_free(pe_resource_t * rsc);
	void clone_free(pe_resource_t * rsc);
	void pe__free_bundle(pe_resource_t *rsc);

	enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
	enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
	gboolean current);

	void pe__count_common(pe_resource_t *rsc);
	void pe__count_bundle(pe_resource_t *rsc);

	gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
	pe_working_set_t * data_set);
	void common_free(pe_resource_t * rsc);

	pe_node_t pe__copy_node(const pe_node_t this_node);
	extern time_t get_effective_time(pe_working_set_t * data_set);

	/* Failure handling utilities (from failcounts.c) */

	// bit flags for fail count handling options
	enum pe_fc_flags_e {
	pe_fc_default = 0x00,
	pe_fc_effective = 0x01, // don't count expired failures
	pe_fc_fillers = 0x02, // if container, include filler failures in count
	};

	int pe_get_failcount(pe_node_t node, pe_resource_t rsc, time_t *last_failure,
	uint32_t flags, xmlNode *xml_op,
	pe_working_set_t *data_set);

	pe_action_t pe__clear_failcount(pe_resource_t rsc, pe_node_t *node,
	const char *reason,
	pe_working_set_t *data_set);

	/* Functions for finding/counting a resource's active nodes */

	pe_node_t pe__find_active_on(const pe_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);
	pe_node_t pe__find_active_requires(const pe_resource_t rsc,
	unsigned int *count);

	static inline pe_node_t *
	pe__current_node(const pe_resource_t *rsc)
	{
	return pe__find_active_on(rsc, NULL, NULL);
	}


	/* Binary like operators for lists of nodes */
	extern void node_list_exclude(GHashTable * list, GListPtr list2, gboolean merge_scores);

	GHashTable pe__node_list2table(GList list);

	static inline gpointer
	pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
	{
	if (hash) {
	return g_hash_table_lookup(hash, key);
	}
	return NULL;
	}

	extern pe_action_t get_pseudo_op(const char name, pe_working_set_t * data_set);
	extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);

	/* Printing functions for debug */
	extern void print_node(const char pre_text, pe_node_t node, gboolean details);
	extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
	extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);

	extern void dump_node_capacity(int level, const char comment, pe_node_t node);
	extern void dump_rsc_utilization(int level, const char comment, pe_resource_t rsc, pe_node_t * node);

	void pe__show_node_weights_as(const char file, const char function,
	int line, bool to_log, pe_resource_t *rsc,
	const char comment, GHashTable nodes);

	#define pe__show_node_weights(level, rsc, text, nodes) \
	pe__show_node_weights_as(__FILE__, __FUNCTION__, __LINE__, \
	(level), (rsc), (text), (nodes))

	/* Sorting functions */
	extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
	extern gint sort_rsc_index(gconstpointer a, gconstpointer b);

	extern xmlNode find_rsc_op_entry(pe_resource_t rsc, const char *key);

	extern pe_action_t custom_action(pe_resource_t rsc, char key, const char task, pe_node_t * on_node,
	gboolean optional, gboolean foo, pe_working_set_t * data_set);

	# define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
	# define delete_action(rsc, node, optional) custom_action( \
	rsc, delete_key(rsc), CRMD_ACTION_DELETE, node, \
	optional, TRUE, data_set);

	# define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
	# define stopped_action(rsc, node, optional) custom_action( \
	rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node, \
	optional, TRUE, data_set);

	# define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
	# define stop_action(rsc, node, optional) custom_action( \
	rsc, stop_key(rsc), CRMD_ACTION_STOP, node, \
	optional, TRUE, data_set);

	# define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
	# define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
	# define start_action(rsc, node, optional) custom_action( \
	rsc, start_key(rsc), CRMD_ACTION_START, node, \
	optional, TRUE, data_set)

	# define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
	# define started_action(rsc, node, optional) custom_action( \
	rsc, started_key(rsc), CRMD_ACTION_STARTED, node, \
	optional, TRUE, data_set)

	# define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
	# define promote_action(rsc, node, optional) custom_action( \
	rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node, \
	optional, TRUE, data_set)

	# define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
	# define promoted_action(rsc, node, optional) custom_action( \
	rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node, \
	optional, TRUE, data_set)

	# define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
	# define demote_action(rsc, node, optional) custom_action( \
	rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node, \
	optional, TRUE, data_set)

	# define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
	# define demoted_action(rsc, node, optional) custom_action( \
	rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node, \
	optional, TRUE, data_set)

	extern int pe_get_configured_timeout(pe_resource_t rsc, const char action,
	pe_working_set_t *data_set);

	extern pe_action_t find_first_action(GListPtr input, const char uuid, const char *task,
	pe_node_t * on_node);
	extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
	gboolean allow_non_atomic);

	extern GListPtr find_actions(GListPtr input, const char key, const pe_node_t on_node);
	GList find_actions_exact(GList input, const char *key,
	const pe_node_t *on_node);
	extern GListPtr find_recurring_actions(GListPtr input, pe_node_t * not_on_node);
	GList pe__resource_actions(const pe_resource_t rsc, const pe_node_t *node,
	const char *task, bool require_node);

	extern void pe_free_action(pe_action_t * action);

	extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
	pe_working_set_t * data_set);

	extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
	extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);

	extern pe_resource_t find_clone_instance(pe_resource_t rsc, const char *sub_id,
	pe_working_set_t * data_set);

	extern void destroy_ticket(gpointer data);
	extern pe_ticket_t ticket_new(const char ticket_id, pe_working_set_t * data_set);

	// Resources for manipulating resource names
	const char pe_base_name_end(const char id);
	char clone_strip(const char last_rsc_id);
	char clone_zero(const char last_rsc_id);

	static inline bool
	pe_base_name_eq(pe_resource_t rsc, const char id)
	{
	if (id && rsc && rsc->id) {
	// Number of characters in rsc->id before any clone suffix
	size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;

	return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
	}
	return FALSE;
	}

	int pe__target_rc_from_xml(xmlNode *xml_op);

	gint sort_node_uname(gconstpointer a, gconstpointer b);
	bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);

	enum rsc_digest_cmp_val {
	/! Digests are the same /
	RSC_DIGEST_MATCH = 0,
	/! Params that require a restart changed /
	RSC_DIGEST_RESTART,
	/! Some parameter changed. /
	RSC_DIGEST_ALL,
	/*! rsc op didn't have a digest associated with it, so
	* it is unknown if parameters changed or not. */
	RSC_DIGEST_UNKNOWN,
	};

	typedef struct op_digest_cache_s {
	enum rsc_digest_cmp_val rc;
	xmlNode *params_all;
	xmlNode *params_secure;
	xmlNode *params_restart;
	char *digest_all_calc;
	char *digest_secure_calc;
	char *digest_restart_calc;
	} op_digest_cache_t;

	op_digest_cache_t rsc_action_digest_cmp(pe_resource_t rsc, xmlNode * xml_op, pe_node_t * node,
	pe_working_set_t * data_set);

	pe_action_t pe_fence_op(pe_node_t node, const char op, bool optional, const char reason, bool priority_delay, pe_working_set_t * data_set);
	void trigger_unfencing(
	pe_resource_t * rsc, pe_node_t node, const char reason, pe_action_t dependency, pe_working_set_t data_set);

	void pe_action_set_reason(pe_action_t action, const char reason, bool overwrite);
	void pe_action_set_flag_reason(const char function, long line, pe_action_t action, pe_action_t reason, const char text, enum pe_action_flags flags, bool overwrite);

	#define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE)
	#define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE)

	void pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);
	void pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags);

	gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref);

	void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
	void * print_data, gboolean print_all);
	int pe__rscs_brief_output(pcmk__output_t *out, GListPtr rsc_list, long options, gboolean print_all);
	void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);

	pe_node_t pe_create_node(const char id, const char uname, const char type,
	const char score, pe_working_set_t data_set);
	void common_print(pe_resource_t * rsc, const char pre_text, const char name, pe_node_t node, long options, void print_data);
	int pe__common_output_text(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, long options);
	int pe__common_output_html(pcmk__output_t out, pe_resource_t rsc, const char name, pe_node_t node, long options);
	pe_resource_t pe__find_bundle_replica(const pe_resource_t bundle,
	const pe_node_t *node);
	bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
	const char pe__add_bundle_remote_name(pe_resource_t rsc, xmlNode *xml,
	const char *field);
	const char pe_node_attribute_calculated(const pe_node_t node,
	const char *name,
	const pe_resource_t *rsc);
	const char pe_node_attribute_raw(pe_node_t node, const char *name);
	bool pe__is_universal_clone(pe_resource_t *rsc,
	pe_working_set_t *data_set);
	void pe__add_param_check(xmlNode rsc_op, pe_resource_t rsc, pe_node_t *node,
	enum pe_check_parameters, pe_working_set_t *data_set);
	void pe__foreach_param_check(pe_working_set_t *data_set,
	void (cb)(pe_resource_t, pe_node_t, xmlNode,
	enum pe_check_parameters,
	pe_working_set_t*));
	void pe__free_param_checks(pe_working_set_t *data_set);

	bool pe__shutdown_requested(pe_node_t *node);
	void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);

	/*!
	* \internal
	* \brief Register xml formatting message functions.
	*/
	void pe__register_messages(pcmk__output_t *out);

	void pe__unpack_dataset_nvpairs(xmlNode xml_obj, const char set_name,
	pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	pe_working_set_t *data_set);

	bool pe__resource_is_disabled(pe_resource_t *rsc);
	pe_action_t pe__clear_resource_history(pe_resource_t rsc, pe_node_t *node,
	pe_working_set_t *data_set);

	GListPtr pe__rscs_with_tag(pe_working_set_t data_set, const char tag_name);
	GListPtr pe__unames_with_tag(pe_working_set_t data_set, const char tag_name);
	bool pe__rsc_has_tag(pe_working_set_t data_set, const char rsc, const char *tag);
	bool pe__uname_has_tag(pe_working_set_t data_set, const char node, const char *tag);

	bool pe__rsc_running_on_any_node_in_list(pe_resource_t *rsc, GListPtr node_list);
	GListPtr pe__filter_rsc_list(GListPtr rscs, GListPtr filter);

	bool pcmk__rsc_filtered_by_node(pe_resource_t *rsc, GListPtr only_node);

	gboolean pe__bundle_is_filtered(pe_resource_t *rsc, GListPtr only_rsc, gboolean check_parent);
	gboolean pe__clone_is_filtered(pe_resource_t *rsc, GListPtr only_rsc, gboolean check_parent);
	gboolean pe__group_is_filtered(pe_resource_t *rsc, GListPtr only_rsc, gboolean check_parent);
	gboolean pe__native_is_filtered(pe_resource_t *rsc, GListPtr only_rsc, gboolean check_parent);

	#endif
	diff --git a/lib/common/utils.c b/lib/common/utils.c
	index 35e168dd5b..c53c9fcc2d 100644
	--- a/lib/common/utils.c
	+++ b/lib/common/utils.c
	@@ -1,616 +1,616 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <pwd.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/common/ipc.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/mainloop.h>
	#include <libxml2/libxml/relaxng.h>

	#ifndef PW_BUFFER_LEN
	# define PW_BUFFER_LEN 500
	#endif

	CRM_TRACE_INIT_DATA(common);

	-gboolean crm_config_error = FALSE;
	-gboolean crm_config_warning = FALSE;
	+bool pcmk__config_error = false;
	+bool pcmk__config_warning = false;
	char *crm_system_name = NULL;

	int pcmk__score_red = 0;
	int pcmk__score_green = 0;
	int pcmk__score_yellow = 0;

	int
	char2score(const char *score)
	{
	int score_f = 0;

	if (score == NULL) {

	} else if (pcmk_str_is_minus_infinity(score)) {
	score_f = -CRM_SCORE_INFINITY;

	} else if (pcmk_str_is_infinity(score)) {
	score_f = CRM_SCORE_INFINITY;

	} else if (pcmk__str_eq(score, "red", pcmk__str_casei)) {
	score_f = pcmk__score_red;

	} else if (pcmk__str_eq(score, "yellow", pcmk__str_casei)) {
	score_f = pcmk__score_yellow;

	} else if (pcmk__str_eq(score, "green", pcmk__str_casei)) {
	score_f = pcmk__score_green;

	} else {
	score_f = crm_parse_int(score, NULL);
	if (score_f > 0 && score_f > CRM_SCORE_INFINITY) {
	score_f = CRM_SCORE_INFINITY;

	} else if (score_f < 0 && score_f < -CRM_SCORE_INFINITY) {
	score_f = -CRM_SCORE_INFINITY;
	}
	}

	return score_f;
	}

	char *
	score2char_stack(int score, char *buf, size_t len)
	{
	if (score >= CRM_SCORE_INFINITY) {
	strncpy(buf, CRM_INFINITY_S, 9);
	} else if (score <= -CRM_SCORE_INFINITY) {
	strncpy(buf, CRM_MINUS_INFINITY_S , 10);
	} else {
	return crm_itoa_stack(score, buf, len);
	}

	return buf;
	}

	char *
	score2char(int score)
	{
	if (score >= CRM_SCORE_INFINITY) {
	return strdup(CRM_INFINITY_S);

	} else if (score <= -CRM_SCORE_INFINITY) {
	return strdup(CRM_MINUS_INFINITY_S);
	}
	return crm_itoa(score);
	}

	int
	crm_user_lookup(const char name, uid_t uid, gid_t * gid)
	{
	int rc = pcmk_ok;
	char *buffer = NULL;
	struct passwd pwd;
	struct passwd *pwentry = NULL;

	buffer = calloc(1, PW_BUFFER_LEN);
	if (buffer == NULL) {
	return -ENOMEM;
	}

	rc = getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
	if (pwentry) {
	if (uid) {
	*uid = pwentry->pw_uid;
	}
	if (gid) {
	*gid = pwentry->pw_gid;
	}
	crm_trace("User %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);

	} else {
	rc = rc? -rc : -EINVAL;
	crm_info("User %s lookup: %s", name, pcmk_strerror(rc));
	}

	free(buffer);
	return rc;
	}

	/*!
	* \brief Get user and group IDs of pacemaker daemon user
	*
	* \param[out] uid If non-NULL, where to store daemon user ID
	* \param[out] gid If non-NULL, where to store daemon group ID
	*
	* \return pcmk_ok on success, -errno otherwise
	*/
	int
	pcmk_daemon_user(uid_t uid, gid_t gid)
	{
	static uid_t daemon_uid;
	static gid_t daemon_gid;
	static bool found = false;
	int rc = pcmk_err_generic;

	if (!found) {
	rc = crm_user_lookup(CRM_DAEMON_USER, &daemon_uid, &daemon_gid);
	if (rc == pcmk_ok) {
	found = true;
	}
	}
	if (found) {
	if (uid) {
	*uid = daemon_uid;
	}
	if (gid) {
	*gid = daemon_gid;
	}
	}
	return rc;
	}

	static int
	crm_version_helper(const char text, const char *end_text)
	{
	int atoi_result = -1;

	CRM_ASSERT(end_text != NULL);

	errno = 0;

	if (text != NULL && text[0] != 0) {
	/* seemingly sacrificing const-correctness -- because while strtol
	doesn't modify the input, it doesn't want to artificially taint the
	"end_text" pointer-to-pointer-to-first-char-in-string with constness
	in case the input wasn't actually constant -- by semantic definition
	not a single character will get modified so it shall be perfectly
	safe to make compiler happy with dropping "const" qualifier here */
	atoi_result = (int) strtol(text, (char **) end_text, 10);

	if (errno == EINVAL) {
	crm_err("Conversion of '%s' %c failed", text, text[0]);
	atoi_result = -1;
	}
	}
	return atoi_result;
	}

	/*
	* version1 < version2 : -1
	* version1 = version2 : 0
	* version1 > version2 : 1
	*/
	int
	compare_version(const char version1, const char version2)
	{
	int rc = 0;
	int lpc = 0;
	const char ver1_iter, ver2_iter;

	if (version1 == NULL && version2 == NULL) {
	return 0;
	} else if (version1 == NULL) {
	return -1;
	} else if (version2 == NULL) {
	return 1;
	}

	ver1_iter = version1;
	ver2_iter = version2;

	while (1) {
	int digit1 = 0;
	int digit2 = 0;

	lpc++;

	if (ver1_iter == ver2_iter) {
	break;
	}

	if (ver1_iter != NULL) {
	digit1 = crm_version_helper(ver1_iter, &ver1_iter);
	}

	if (ver2_iter != NULL) {
	digit2 = crm_version_helper(ver2_iter, &ver2_iter);
	}

	if (digit1 < digit2) {
	rc = -1;
	break;

	} else if (digit1 > digit2) {
	rc = 1;
	break;
	}

	if (ver1_iter != NULL && *ver1_iter == '.') {
	ver1_iter++;
	}
	if (ver1_iter != NULL && *ver1_iter == '\0') {
	ver1_iter = NULL;
	}

	if (ver2_iter != NULL && *ver2_iter == '.') {
	ver2_iter++;
	}
	if (ver2_iter != NULL && *ver2_iter == 0) {
	ver2_iter = NULL;
	}
	}

	if (rc == 0) {
	crm_trace("%s == %s (%d)", version1, version2, lpc);
	} else if (rc < 0) {
	crm_trace("%s < %s (%d)", version1, version2, lpc);
	} else if (rc > 0) {
	crm_trace("%s > %s (%d)", version1, version2, lpc);
	}

	return rc;
	}

	/*!
	* \brief Parse milliseconds from a Pacemaker interval specification
	*
	* \param[in] input Pacemaker time interval specification (a bare number of
	* seconds, a number with a unit optionally with whitespace
	* before and/or after the number, or an ISO 8601 duration)
	*
	* \return Milliseconds equivalent of given specification on success (limited
	* to the range of an unsigned integer), 0 if input is NULL,
	* or 0 (and set errno to EINVAL) on error
	*/
	guint
	crm_parse_interval_spec(const char *input)
	{
	long long msec = -1;

	errno = 0;
	if (input == NULL) {
	return 0;

	} else if (input[0] == 'P') {
	crm_time_t *period_s = crm_time_parse_duration(input);

	if (period_s) {
	msec = 1000 * crm_time_get_seconds(period_s);
	crm_time_free(period_s);
	}

	} else {
	msec = crm_get_msec(input);
	}

	if (msec < 0) {
	crm_warn("Using 0 instead of '%s'", input);
	errno = EINVAL;
	return 0;
	}
	return (msec >= G_MAXUINT)? G_MAXUINT : (guint) msec;
	}

	extern bool crm_is_daemon;

	/* coverity[+kill] */
	void
	crm_abort(const char file, const char function, int line,
	const char *assert_condition, gboolean do_core, gboolean do_fork)
	{
	int rc = 0;
	int pid = 0;
	int status = 0;

	/* Implied by the parent's error logging below */
	/* crm_write_blackbox(0); */

	if(crm_is_daemon == FALSE) {
	/* This is a command line tool - do not fork */

	/* crm_add_logfile(NULL); * Record it to a file? */
	crm_enable_stderr(TRUE); /* Make sure stderr is enabled so we can tell the caller */
	do_fork = FALSE; /* Just crash if needed */
	}

	if (do_core == FALSE) {
	crm_err("%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition);
	return;

	} else if (do_fork) {
	pid = fork();

	} else {
	crm_err("%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition);
	}

	if (pid == -1) {
	crm_crit("%s: Cannot create core for non-fatal assert at %s:%d : %s",
	function, file, line, assert_condition);
	return;

	} else if(pid == 0) {
	/* Child process */
	abort();
	return;
	}

	/* Parent process */
	crm_err("%s: Forked child %d to record non-fatal assert at %s:%d : %s",
	function, pid, file, line, assert_condition);
	crm_write_blackbox(SIGTRAP, NULL);

	do {
	rc = waitpid(pid, &status, 0);
	if(rc == pid) {
	return; /* Job done */
	}

	} while(errno == EINTR);

	if (errno == ECHILD) {
	/* crm_mon does this */
	crm_trace("Cannot wait on forked child %d - SIGCHLD is probably set to SIG_IGN", pid);
	return;
	}
	crm_perror(LOG_ERR, "Cannot wait on forked child %d", pid);
	}

	void
	crm_make_daemon(const char name, gboolean daemonize, const char pidfile)
	{
	int rc;
	pid_t pid;

	if (daemonize == FALSE) {
	return;
	}

	/* Check before we even try... */
	rc = pcmk__pidfile_matches(pidfile, 1, name, &pid);
	if ((rc != pcmk_rc_ok) && (rc != ENOENT)) {
	crm_err("%s: already running [pid %lld in %s]",
	name, (long long) pid, pidfile);
	printf("%s: already running [pid %lld in %s]\n",
	name, (long long) pid, pidfile);
	crm_exit(CRM_EX_ERROR);
	}

	pid = fork();
	if (pid < 0) {
	fprintf(stderr, "%s: could not start daemon\n", name);
	crm_perror(LOG_ERR, "fork");
	crm_exit(CRM_EX_OSERR);

	} else if (pid > 0) {
	crm_exit(CRM_EX_OK);
	}

	rc = pcmk__lock_pidfile(pidfile, name);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not lock '%s' for %s: %s " CRM_XS " rc=%d",
	pidfile, name, pcmk_rc_str(rc), rc);
	printf("Could not lock '%s' for %s: %s (%d)\n",
	pidfile, name, pcmk_rc_str(rc), rc);
	crm_exit(CRM_EX_ERROR);
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	close(STDIN_FILENO);
	pcmk__open_devnull(O_RDONLY); // stdin (fd 0)

	close(STDOUT_FILENO);
	pcmk__open_devnull(O_WRONLY); // stdout (fd 1)

	close(STDERR_FILENO);
	pcmk__open_devnull(O_WRONLY); // stderr (fd 2)
	}

	char *
	crm_meta_name(const char *field)
	{
	int lpc = 0;
	int max = 0;
	char *crm_name = NULL;

	CRM_CHECK(field != NULL, return NULL);
	crm_name = crm_strdup_printf(CRM_META "_%s", field);

	/* Massage the names so they can be used as shell variables */
	max = strlen(crm_name);
	for (; lpc < max; lpc++) {
	switch (crm_name[lpc]) {
	case '-':
	crm_name[lpc] = '_';
	break;
	}
	}
	return crm_name;
	}

	const char *
	crm_meta_value(GHashTable * hash, const char *field)
	{
	char *key = NULL;
	const char *value = NULL;

	key = crm_meta_name(field);
	if (key) {
	value = g_hash_table_lookup(hash, key);
	free(key);
	}

	return value;
	}

	#ifdef HAVE_UUID_UUID_H
	# include <uuid/uuid.h>
	#endif

	char *
	crm_generate_uuid(void)
	{
	unsigned char uuid[16];
	char buffer = malloc(37); / Including NUL byte */

	uuid_generate(uuid);
	uuid_unparse(uuid, buffer);
	return buffer;
	}

	/*!
	* \brief Get name to be used as identifier for cluster messages
	*
	* \param[in] name Actual system name to check
	*
	* \return Non-NULL cluster message identifier corresponding to name
	*
	* \note The Pacemaker daemons were renamed in version 2.0.0, but the old names
	* must continue to be used as the identifier for cluster messages, so
	* that mixed-version clusters are possible during a rolling upgrade.
	*/
	const char *
	pcmk_message_name(const char *name)
	{
	if (name == NULL) {
	return "unknown";

	} else if (!strcmp(name, "pacemaker-attrd")) {
	return "attrd";

	} else if (!strcmp(name, "pacemaker-based")) {
	return CRM_SYSTEM_CIB;

	} else if (!strcmp(name, "pacemaker-controld")) {
	return CRM_SYSTEM_CRMD;

	} else if (!strcmp(name, "pacemaker-execd")) {
	return CRM_SYSTEM_LRMD;

	} else if (!strcmp(name, "pacemaker-fenced")) {
	return "stonith-ng";

	} else if (!strcmp(name, "pacemaker-schedulerd")) {
	return CRM_SYSTEM_PENGINE;

	} else {
	return name;
	}
	}

	/*!
	* \brief Check whether a string represents a cluster daemon name
	*
	* \param[in] name String to check
	*
	* \return TRUE if name is standard client name used by daemons, FALSE otherwise
	*/
	bool
	crm_is_daemon_name(const char *name)
	{
	name = pcmk_message_name(name);
	return (!strcmp(name, CRM_SYSTEM_CRMD)
	\|\| !strcmp(name, CRM_SYSTEM_STONITHD)
	\|\| !strcmp(name, "stonith-ng")
	\|\| !strcmp(name, "attrd")
	\|\| !strcmp(name, CRM_SYSTEM_CIB)
	\|\| !strcmp(name, CRM_SYSTEM_MCP)
	\|\| !strcmp(name, CRM_SYSTEM_DC)
	\|\| !strcmp(name, CRM_SYSTEM_TENGINE)
	\|\| !strcmp(name, CRM_SYSTEM_LRMD));
	}

	#include <md5.h>

	char *
	crm_md5sum(const char *buffer)
	{
	int lpc = 0, len = 0;
	char *digest = NULL;
	unsigned char raw_digest[MD5_DIGEST_SIZE];

	if (buffer == NULL) {
	buffer = "";
	}
	len = strlen(buffer);

	crm_trace("Beginning digest of %d bytes", len);
	digest = malloc(2 * MD5_DIGEST_SIZE + 1);
	if(digest) {
	md5_buffer(buffer, len, raw_digest);
	for (lpc = 0; lpc < MD5_DIGEST_SIZE; lpc++) {
	sprintf(digest + (2 * lpc), "%02x", raw_digest[lpc]);
	}
	digest[(2 * MD5_DIGEST_SIZE)] = 0;
	crm_trace("Digest %s.", digest);

	} else {
	crm_err("Could not create digest");
	}
	return digest;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	void
	crm_gnutls_global_init(void)
	{
	signal(SIGPIPE, SIG_IGN);
	gnutls_global_init();
	}
	#endif

	/*!
	* \brief Get the local hostname
	*
	* \return Newly allocated string with name, or NULL (and set errno) on error
	*/
	char *
	pcmk_hostname()
	{
	struct utsname hostinfo;

	return (uname(&hostinfo) < 0)? NULL : strdup(hostinfo.nodename);
	}

	bool
	pcmk_str_is_infinity(const char *s) {
	return pcmk__str_any_of(s, CRM_INFINITY_S, CRM_PLUS_INFINITY_S, NULL);
	}

	bool
	pcmk_str_is_minus_infinity(const char *s) {
	return pcmk__str_eq(s, CRM_MINUS_INFINITY_S, pcmk__str_none);
	}
	diff --git a/lib/pacemaker/pcmk_sched_allocate.c b/lib/pacemaker/pcmk_sched_allocate.c
	index eeedb7611b..f3877f87a6 100644
	--- a/lib/pacemaker/pcmk_sched_allocate.c
	+++ b/lib/pacemaker/pcmk_sched_allocate.c
	@@ -1,3059 +1,3059 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	CRM_TRACE_INIT_DATA(pacemaker);

	void set_alloc_actions(pe_working_set_t * data_set);
	extern void ReloadRsc(pe_resource_t * rsc, pe_node_t node, pe_working_set_t data_set);
	extern gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
	static void apply_remote_node_ordering(pe_working_set_t *data_set);
	static enum remote_connection_state get_remote_node_state(pe_node_t *node);

	enum remote_connection_state {
	remote_state_unknown = 0,
	remote_state_alive = 1,
	remote_state_resting = 2,
	remote_state_failed = 3,
	remote_state_stopped = 4
	};

	static const char *
	state2text(enum remote_connection_state state)
	{
	switch (state) {
	case remote_state_unknown:
	return "unknown";
	case remote_state_alive:
	return "alive";
	case remote_state_resting:
	return "resting";
	case remote_state_failed:
	return "failed";
	case remote_state_stopped:
	return "stopped";
	}

	return "impossible";
	}

	resource_alloc_functions_t resource_class_alloc_functions[] = {
	{
	pcmk__native_merge_weights,
	pcmk__native_allocate,
	native_create_actions,
	native_create_probe,
	native_internal_constraints,
	native_rsc_colocation_lh,
	native_rsc_colocation_rh,
	native_rsc_location,
	native_action_flags,
	native_update_actions,
	native_expand,
	native_append_meta,
	},
	{
	pcmk__group_merge_weights,
	pcmk__group_allocate,
	group_create_actions,
	native_create_probe,
	group_internal_constraints,
	group_rsc_colocation_lh,
	group_rsc_colocation_rh,
	group_rsc_location,
	group_action_flags,
	group_update_actions,
	group_expand,
	group_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__clone_allocate,
	clone_create_actions,
	clone_create_probe,
	clone_internal_constraints,
	clone_rsc_colocation_lh,
	clone_rsc_colocation_rh,
	clone_rsc_location,
	clone_action_flags,
	pcmk__multi_update_actions,
	clone_expand,
	clone_append_meta,
	},
	{
	pcmk__native_merge_weights,
	pcmk__bundle_allocate,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_rsc_colocation_lh,
	pcmk__bundle_rsc_colocation_rh,
	pcmk__bundle_rsc_location,
	pcmk__bundle_action_flags,
	pcmk__multi_update_actions,
	pcmk__bundle_expand,
	pcmk__bundle_append_meta,
	}
	};

	gboolean
	update_action_flags(pe_action_t * action, enum pe_action_flags flags, const char *source, int line)
	{
	static unsigned long calls = 0;
	gboolean changed = FALSE;
	gboolean clear = pcmk_is_set(flags, pe_action_clear);
	enum pe_action_flags last = action->flags;

	if (clear) {
	pe__clear_action_flags_as(source, line, action, flags);
	} else {
	pe__set_action_flags_as(source, line, action, flags);
	}

	if (last != action->flags) {
	calls++;
	changed = TRUE;
	/* Useful for tracking down _who_ changed a specific flag */
	/* CRM_ASSERT(calls != 534); */
	pe__clear_raw_action_flags(flags, "action update", pe_action_clear);
	crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
	action->uuid, action->node ? action->node->details->uname : "[none]",
	clear ? "un-" : "", flags, last, action->flags, calls, source);
	}

	return changed;
	}

	static gboolean
	check_rsc_parameters(pe_resource_t * rsc, pe_node_t * node, xmlNode * rsc_entry,
	gboolean active_here, pe_working_set_t * data_set)
	{
	int attr_lpc = 0;
	gboolean force_restart = FALSE;
	gboolean delete_resource = FALSE;
	gboolean changed = FALSE;

	const char *value = NULL;
	const char *old_value = NULL;

	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
	value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
	old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
	if (value == old_value /* i.e. NULL */
	\|\| pcmk__str_eq(value, old_value, pcmk__str_none)) {
	continue;
	}

	changed = TRUE;
	trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
	if (active_here) {
	force_restart = TRUE;
	crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
	rsc->id, node->details->uname, attr_list[attr_lpc],
	crm_str(old_value), crm_str(value));
	}
	}
	if (force_restart) {
	/* make sure the restart happens */
	stop_action(rsc, node, FALSE);
	pe__set_resource_flags(rsc, pe_rsc_start_pending);
	delete_resource = TRUE;

	} else if (changed) {
	delete_resource = TRUE;
	}
	return delete_resource;
	}

	static void
	CancelXmlOp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * active_node,
	const char reason, pe_working_set_t data_set)
	{
	guint interval_ms = 0;
	pe_action_t *cancel = NULL;

	const char *task = NULL;
	const char *call_id = NULL;

	CRM_CHECK(xml_op != NULL, return);
	CRM_CHECK(active_node != NULL, return);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	crm_info("Action " PCMK__OP_FMT " on %s will be stopped: %s",
	rsc->id, task, interval_ms,
	active_node->details->uname, (reason? reason : "unknown"));

	cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
	custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
	}

	static gboolean
	check_action_definition(pe_resource_t * rsc, pe_node_t * active_node, xmlNode * xml_op,
	pe_working_set_t * data_set)
	{
	char *key = NULL;
	guint interval_ms = 0;
	const op_digest_cache_t *digest_data = NULL;
	gboolean did_change = FALSE;

	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_secure = NULL;

	CRM_CHECK(active_node != NULL, return FALSE);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms > 0) {
	xmlNode *op_match = NULL;

	/* we need to reconstruct the key because of the way we used to construct resource IDs */
	key = pcmk__op_key(rsc->id, task, interval_ms);

	pe_rsc_trace(rsc, "Checking parameters for %s", key);
	op_match = find_rsc_op_entry(rsc, key);

	if ((op_match == NULL)
	&& pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)) {
	CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
	free(key);
	return TRUE;

	} else if (op_match == NULL) {
	pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
	free(key);
	return TRUE;
	}
	free(key);
	key = NULL;
	}

	crm_trace("Testing " PCMK__OP_FMT " on %s",
	rsc->id, task, interval_ms, active_node->details->uname);
	if ((interval_ms == 0) && pcmk__str_eq(task, RSC_STATUS, pcmk__str_casei)) {
	/* Reload based on the start action not a probe */
	task = RSC_START;

	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_MIGRATED, pcmk__str_casei)) {
	/* Reload based on the start action not a migrate */
	task = RSC_START;
	} else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_PROMOTE, pcmk__str_casei)) {
	/* Reload based on the start action not a promote */
	task = RSC_START;
	}

	digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
	}

	if(digest_data->rc != RSC_DIGEST_MATCH
	&& digest_secure
	&& digest_data->digest_secure_calc
	&& strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
	if (pcmk_is_set(data_set->flags, pe_flag_stdout)) {
	printf("Only 'private' parameters to " PCMK__OP_FMT
	" on %s changed: %s\n",
	rsc->id, task, interval_ms, active_node->details->uname,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	}

	} else if (digest_data->rc == RSC_DIGEST_RESTART) {
	/* Changes that force a restart */
	pe_action_t *required = NULL;

	did_change = TRUE;
	key = pcmk__op_key(rsc->id, task, interval_ms);
	crm_log_xml_info(digest_data->params_restart, "params:restart");
	required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
	pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
	"resource definition change", pe_action_optional, TRUE);

	trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);

	} else if ((digest_data->rc == RSC_DIGEST_ALL) \|\| (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
	/* Changes that can potentially be handled by a reload */
	const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	did_change = TRUE;
	trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
	crm_log_xml_info(digest_data->params_all, "params:reload");
	key = pcmk__op_key(rsc->id, task, interval_ms);

	if (interval_ms > 0) {
	pe_action_t *op = NULL;

	#if 0
	/* Always reload/restart the entire resource */
	ReloadRsc(rsc, active_node, data_set);
	#else
	/* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
	op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
	pe__set_action_flags(op, pe_action_reschedule);
	#endif

	} else if (digest_restart) {
	pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);

	/* Reload this resource */
	ReloadRsc(rsc, active_node, data_set);
	free(key);

	} else {
	pe_action_t *required = NULL;
	pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);

	/* Re-send the start/demote/promote op
	* Recurring ops will be detected independently
	*/
	required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
	pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
	"resource definition change", pe_action_optional, TRUE);
	}
	}

	return did_change;
	}

	/*!
	* \internal
	* \brief Do deferred action checks after allocation
	*
	* \param[in] data_set Working set for cluster
	*/
	static void
	check_params(pe_resource_t rsc, pe_node_t node, xmlNode *rsc_op,
	enum pe_check_parameters check, pe_working_set_t *data_set)
	{
	const char *reason = NULL;
	op_digest_cache_t *digest_data = NULL;

	switch (check) {
	case pe_check_active:
	if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	reason = "action definition changed";
	}
	break;

	case pe_check_last_failure:
	digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
	switch (digest_data->rc) {
	case RSC_DIGEST_UNKNOWN:
	crm_trace("Resource %s history entry %s on %s has no digest to compare",
	rsc->id, ID(rsc_op), node->details->id);
	break;
	case RSC_DIGEST_MATCH:
	break;
	default:
	reason = "resource parameters have changed";
	break;
	}
	break;
	}

	if (reason) {
	pe__clear_failcount(rsc, node, reason, data_set);
	}
	}

	static void
	check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;

	const char *task = NULL;

	xmlNode *rsc_op = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	CRM_CHECK(node != NULL, return);

	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	pe_resource_t *parent = uber_parent(rsc);
	if(parent == NULL
	\|\| pe_rsc_is_clone(parent) == FALSE
	\|\| pcmk_is_set(parent->flags, pe_rsc_unique)) {
	pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
	DeleteRsc(rsc, node, FALSE, data_set);
	} else {
	pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
	}
	return;

	} else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}
	pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
	rsc->id, node->details->uname);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);

	if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}

	for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL;
	rsc_op = __xml_next_element(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;
	guint interval_ms = 0;

	offset++;

	if (start_index < stop_index) {
	/* stopped */
	continue;
	} else if (offset < start_index) {
	/* action occurred prior to a start */
	continue;
	}

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);

	if ((interval_ms > 0) &&
	(pcmk_is_set(rsc->flags, pe_rsc_maintenance) \|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);

	} else if ((interval_ms > 0) \|\| pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
	RSC_PROMOTE, RSC_MIGRATED, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pe_check_active,
	data_set);

	} else if (check_action_definition(rsc, node, rsc_op, data_set)
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	data_set);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	static GListPtr
	find_rsc_list(GListPtr result, pe_resource_t * rsc, const char *id, gboolean renamed_clones,
	gboolean partial, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	gboolean match = FALSE;

	if (id == NULL) {
	return NULL;
	}

	if (rsc == NULL) {
	if (data_set == NULL) {
	return NULL;
	}
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial,
	NULL);
	}
	return result;
	}

	if (partial) {
	if (strstr(rsc->id, id)) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
	match = TRUE;
	}

	} else {
	if (strcmp(rsc->id, id) == 0) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;
	}
	}

	if (match) {
	result = g_list_prepend(result, rsc);
	}

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child = (pe_resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
	}
	}

	return result;
	}

	static void
	check_actions(pe_working_set_t * data_set)
	{
	const char *id = NULL;
	pe_node_t *node = NULL;
	xmlNode *lrm_rscs = NULL;
	xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);

	xmlNode *node_state = NULL;

	for (node_state = __xml_first_child_element(status); node_state != NULL;
	node_state = __xml_next_element(node_state)) {
	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	id = crm_element_value(node_state, XML_ATTR_ID);
	lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);

	node = pe_find_node_id(data_set->nodes, id);

	if (node == NULL) {
	continue;

	/* Still need to check actions for a maintenance node to cancel existing monitor operations */
	} else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
	crm_trace("Skipping param check for %s: can't run resources",
	node->details->uname);
	continue;
	}

	crm_trace("Processing node %s", node->details->uname);
	if (node->details->online
	\|\| pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	xmlNode *rsc_entry = NULL;

	for (rsc_entry = __xml_first_child_element(lrm_rscs);
	rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {

	if (pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	if (xml_has_children(rsc_entry)) {
	GListPtr gIter = NULL;
	GListPtr result = NULL;
	const char *rsc_id = ID(rsc_entry);

	CRM_CHECK(rsc_id != NULL, return);

	result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
	for (gIter = result; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (rsc->variant != pe_native) {
	continue;
	}
	check_actions_for(rsc_entry, rsc, node, data_set);
	}
	g_list_free(result);
	}
	}
	}
	}
	}
	}
	}

	static void
	apply_placement_constraints(pe_working_set_t * data_set)
	{
	for (GList *gIter = data_set->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pe__location_t *cons = gIter->data;

	cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
	}
	}

	static gboolean
	failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
	{
	gboolean rc = FALSE;
	GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);

	if (list) {
	rc = TRUE;
	}
	g_list_free(list);
	return rc;
	}

	/*!
	* \internal
	* \brief Force resource away if failures hit migration threshold
	*
	* \param[in,out] rsc Resource to check for failures
	* \param[in,out] node Node to check for failures
	* \param[in,out] data_set Cluster working set to update
	*/
	static void
	check_migration_threshold(pe_resource_t rsc, pe_node_t node,
	pe_working_set_t *data_set)
	{
	int fail_count, countdown;
	pe_resource_t *failed;

	/* Migration threshold of 0 means never force away */
	if (rsc->migration_threshold == 0) {
	return;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
	return;
	}

	/* If there are no failures, there's no need to force away */
	fail_count = pe_get_failcount(node, rsc, NULL,
	pe_fc_effective\|pe_fc_fillers, NULL,
	data_set);
	if (fail_count <= 0) {
	return;
	}

	/* How many more times recovery will be tried on this node */
	countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);

	/* If failed resource has a parent, we'll force the parent away */
	failed = rsc;
	if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
	failed = uber_parent(rsc);
	}

	if (countdown == 0) {
	resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
	crm_warn("Forcing %s away from %s after %d failures (max=%d)",
	failed->id, node->details->uname, fail_count,
	rsc->migration_threshold);
	} else {
	crm_info("%s can fail %d more times on %s before being forced off",
	failed->id, countdown, node->details->uname);
	}
	}

	static void
	common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	common_apply_stickiness(child_rsc, node, data_set);
	}
	return;
	}

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)
	&& rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
	pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (current == NULL) {

	} else if ((match != NULL)
	\|\| pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	pe_resource_t *sticky_rsc = rsc;

	resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
	pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
	" (node=%s, weight=%d)", sticky_rsc->id,
	node->details->uname, rsc->stickiness);
	} else {
	GHashTableIter iter;
	pe_node_t *nIter = NULL;

	pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
	" and node %s is not explicitly allowed", rsc->id, node->details->uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
	crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
	}
	}
	}

	/* Check the migration threshold only if a failcount clear action
	* has not already been placed for this resource on the node.
	* There is no sense in potentially forcing the resource from this
	* node if the failcount is being reset anyway.
	*
	* @TODO A clear_failcount operation can be scheduled in stage4() via
	* check_actions_for(), or in stage5() via check_params(). This runs in
	* stage2(), so it cannot detect those, meaning we might check the migration
	* threshold when we shouldn't -- worst case, we stop or move the resource,
	* then move it back next transition.
	*/
	if (failcount_clear_action_exists(node, rsc) == FALSE) {
	check_migration_threshold(rsc, node, data_set);
	}
	}

	void
	complex_set_cmds(pe_resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	rsc->cmds = &resource_class_alloc_functions[rsc->variant];

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(child_rsc);
	}
	}

	void
	set_alloc_actions(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	complex_set_cmds(rsc);
	}
	}

	static void
	calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
	{
	const char key = (const char )gKey;
	const char value = (const char )gValue;
	int system_health = (int )user_data;

	if (!gKey \|\| !gValue \|\| !user_data) {
	return;
	}

	if (pcmk__starts_with(key, "#health")) {
	int score;

	/* Convert the value into an integer */
	score = char2score(value);

	/* Add it to the running total */
	system_health = pe__add_scores(score, system_health);
	}
	}

	static gboolean
	apply_system_health(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
	int base_health = 0;

	if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches \| pcmk__str_casei)) {
	/* Prevent any accidental health -> score translation */
	pcmk__score_red = 0;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;
	return TRUE;

	} else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red are
	* weighted away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = 0;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {

	/* Resources on nodes which have health values of red or yellow
	* are forced away from that node.
	*/
	pcmk__score_red = -INFINITY;
	pcmk__score_yellow = -INFINITY;
	pcmk__score_green = 0;

	} else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
	/* Same as the above, but use the r/y/g scores provided by the user
	* Defaults are provided by the pe_prefs table
	* Also, custom health "base score" can be used
	*/
	base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0");

	} else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {

	/* Requires the admin to configure the rsc_location constaints for
	* processing the stored health scores
	*/
	/* TODO: Check for the existence of appropriate node health constraints */
	return TRUE;

	} else {
	crm_err("Unknown node health strategy: %s", health_strategy);
	return FALSE;
	}

	crm_info("Applying automated node health strategy: %s", health_strategy);

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int system_health = base_health;
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Search through the node hash table for system health entries. */
	g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);

	crm_info(" Node %s has an combined system health of %d",
	node->details->uname, system_health);

	/* If the health is non-zero, then create a new rsc2node so that the
	* weight will be added later on.
	*/
	if (system_health != 0) {

	GListPtr gIter2 = data_set->resources;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
	}
	}
	}

	return TRUE;
	}

	gboolean
	stage0(pe_working_set_t * data_set)
	{
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	if (data_set->input == NULL) {
	return FALSE;
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
	crm_trace("Calculating status");
	cluster_status(data_set);
	}

	set_alloc_actions(data_set);
	apply_system_health(data_set);
	unpack_constraints(cib_constraints, data_set);

	return TRUE;
	}

	/*
	* Check nodes for resources started outside of the LRM
	*/
	gboolean
	probe_resources(pe_working_set_t * data_set)
	{
	pe_action_t *probe_node_complete = NULL;

	for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;
	const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);

	if (node->details->online == FALSE) {

	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& (get_remote_node_state(node) == remote_state_failed)) {

	pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) {
	continue;

	} else if (node->details->rsc_discovery_enabled == FALSE) {
	/* resource discovery is disabled for this node */
	continue;
	}

	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pe_action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
	CRM_OP_REPROBE, node, FALSE, TRUE, data_set);

	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	continue;
	}

	for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
	}
	}
	return TRUE;
	}

	static void
	rsc_discover_filter(pe_resource_t rsc, pe_node_t node)
	{
	GListPtr gIter = rsc->children;
	pe_resource_t *top = uber_parent(rsc);
	pe_node_t *match;

	if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;
	rsc_discover_filter(child_rsc, node);
	}

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match && match->rsc_discover_mode != pe_discover_exclusive) {
	match->weight = -INFINITY;
	}
	}

	static time_t
	shutdown_time(pe_node_t node, pe_working_set_t data_set)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown) {
	errno = 0;
	result = (time_t) crm_parse_ll(shutdown, NULL);
	if (errno != 0) {
	result = 0;
	}
	}
	return result? result : get_effective_time(data_set);
	}

	static void
	apply_shutdown_lock(pe_resource_t rsc, pe_working_set_t data_set)
	{
	const char *class;

	// Only primitives and (uncloned) groups may be locked
	if (rsc->variant == pe_group) {
	for (GList *item = rsc->children; item != NULL;
	item = item->next) {
	apply_shutdown_lock((pe_resource_t *) item->data, data_set);
	}
	} else if (rsc->variant != pe_native) {
	return;
	}

	// Fence devices and remote connections can't be locked
	class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc, data_set)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pe_rsc_info(rsc,
	"Cancelling shutdown lock because %s is already active",
	rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node, data_set);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pe_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
	rsc->id, node->details->uname);
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node, data_set);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (data_set->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;

	pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, rsc->lock_node->details->uname,
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, data_set);
	} else {
	pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, rsc->lock_node->details->uname);
	}

	// If resource is locked to one node, ban it from all other nodes
	for (GList *item = data_set->nodes; item != NULL; item = item->next) {
	pe_node_t *node = item->data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
	}
	}
	}

	/*
	* \internal
	* \brief Stage 2 of cluster status: apply node-specific criteria
	*
	* Count known nodes, and apply location constraints, stickiness, and exclusive
	* resource discovery.
	*/
	gboolean
	stage2(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	apply_shutdown_lock((pe_resource_t *) gIter->data, data_set);
	}
	}

	if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
	// @COMPAT API backward compatibility
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node && (node->weight >= 0) && node->details->online
	&& (node->details->type != node_ping)) {
	data_set->max_valid_nodes++;
	}
	}
	}

	crm_trace("Applying placement constraints");
	apply_placement_constraints(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	GListPtr gIter2 = NULL;
	pe_node_t node = (pe_node_t ) gIter->data;

	gIter2 = data_set->resources;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter2->data;

	common_apply_stickiness(rsc, node, data_set);
	rsc_discover_filter(rsc, node);
	}
	}

	return TRUE;
	}

	/*
	* Create internal resource constraints before allocation
	*/
	gboolean
	stage3(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->internal_constraints(rsc, data_set);
	}

	return TRUE;
	}

	/*
	* Check for orphaned or redefined actions
	*/
	gboolean
	stage4(pe_working_set_t * data_set)
	{
	check_actions(data_set);
	return TRUE;
	}

	static void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	static gint
	sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
	{
	int rc = 0;
	int r1_weight = -INFINITY;
	int r2_weight = -INFINITY;

	const char *reason = "existence";

	const GListPtr nodes = (GListPtr) data;
	const pe_resource_t *resource1 = a;
	const pe_resource_t *resource2 = b;

	pe_node_t *r1_node = NULL;
	pe_node_t *r2_node = NULL;
	GListPtr gIter = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;

	reason = "priority";
	r1_weight = resource1->priority;
	r2_weight = resource2->priority;

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	r1_nodes = pcmk__native_merge_weights(convert_const_pointer(resource1),
	resource1->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource1->id, r1_nodes);

	r2_nodes = pcmk__native_merge_weights(convert_const_pointer(resource2),
	resource2->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	pe__show_node_weights(true, NULL, resource2->id, r2_nodes);

	/* Current location score */
	reason = "current location";
	r1_weight = -INFINITY;
	r2_weight = -INFINITY;

	if (resource1->running_on) {
	r1_node = pe__current_node(resource1);
	r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
	if (r1_node != NULL) {
	r1_weight = r1_node->weight;
	}
	}
	if (resource2->running_on) {
	r2_node = pe__current_node(resource2);
	r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
	if (r2_node != NULL) {
	r2_weight = r2_node->weight;
	}
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "score";
	for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	r1_node = NULL;
	r2_node = NULL;

	r1_weight = -INFINITY;
	if (r1_nodes) {
	r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
	}
	if (r1_node) {
	r1_weight = r1_node->weight;
	}

	r2_weight = -INFINITY;
	if (r2_nodes) {
	r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
	}
	if (r2_node) {
	r2_weight = r2_node->weight;
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
	resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
	rc < 0 ? '>' : rc > 0 ? '<' : '=',
	resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);

	if (r1_nodes) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes) {
	g_hash_table_destroy(r2_nodes);
	}

	return rc;
	}

	static void
	allocate_resources(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Allocate remote connection resources first (which will also allocate
	* any colocation dependencies). If the connection is migrating, always
	* prefer the partial migration target.
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == FALSE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
	rsc->id);
	rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
	}
	}

	/* now do the rest of the resources */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;
	if (rsc->is_remote_node == TRUE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating %s resource '%s'",
	crm_element_name(rsc->xml), rsc->id);
	rsc->cmds->allocate(rsc, NULL, data_set);
	}
	}

	/* We always use pe_order_preserve with these convenience functions to exempt
	* internally generated constraints from the prohibition of user constraints
	* involving remote connection resources.
	*
	* The start ordering additionally uses pe_order_runnable_left so that the
	* specified action is not runnable if the start is not runnable.
	*/

	static inline void
	order_start_then_action(pe_resource_t lh_rsc, pe_action_t rh_action,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_rsc && rh_action && data_set) {
	custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
	rh_action->rsc, NULL, rh_action,
	pe_order_preserve \| pe_order_runnable_left \| extra,
	data_set);
	}
	}

	static inline void
	order_action_then_stop(pe_action_t lh_action, pe_resource_t rh_rsc,
	enum pe_ordering extra, pe_working_set_t *data_set)
	{
	if (lh_action && rh_rsc && data_set) {
	custom_action_order(lh_action->rsc, NULL, lh_action,
	rh_rsc, stop_key(rh_rsc), NULL,
	pe_order_preserve \| extra, data_set);
	}
	}

	// Clear fail counts for orphaned rsc on all online nodes
	static void
	cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	if (node->details->online
	&& pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
	data_set)) {

	pe_action_t *clear_op = NULL;

	clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
	data_set);

	/* We can't use order_action_then_stop() here because its
	* pe_order_preserve breaks things
	*/
	custom_action_order(clear_op->rsc, NULL, clear_op,
	rsc, stop_key(rsc), NULL,
	pe_order_optional, data_set);
	}
	}
	}

	gboolean
	stage5(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int log_prio = show_utilization? LOG_STDOUT : LOG_TRACE;

	if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
	GListPtr nodes = g_list_copy(data_set->nodes);

	nodes = sort_nodes_by_weight(nodes, NULL, data_set);
	data_set->resources =
	g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);

	g_list_free(nodes);
	}

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	dump_node_capacity(log_prio, "Original", node);
	}

	crm_trace("Allocating services");
	/* Take (next) highest resource, assign it and create its actions */

	allocate_resources(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	dump_node_capacity(log_prio, "Remaining", node);
	}

	// Process deferred action checks
	pe__foreach_param_check(data_set, check_params);
	pe__free_param_checks(data_set);

	if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Calculating needed probes");
	/* This code probably needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=100:

	With probes:

	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
	ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	36s
	ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph

	Without probes:

	ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
	*/

	probe_resources(data_set);
	}

	crm_trace("Handle orphans");
	if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* There's no need to recurse into rsc->children because those
	* should just be unallocated clone instances.
	*/
	if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
	cleanup_orphans(rsc, data_set);
	}
	}
	}

	crm_trace("Creating actions");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	rsc->cmds->create_actions(rsc, data_set);
	}

	crm_trace("Creating done");
	return TRUE;
	}

	static gboolean
	is_managed(const pe_resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t child_rsc = (pe_resource_t ) gIter->data;

	if (is_managed(child_rsc)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	any_managed_resources(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	if (is_managed(rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Create pseudo-op for guest node fence, and order relative to it
	*
	* \param[in] node Guest node to fence
	* \param[in] data_set Working set of CIB state
	*/
	static void
	fence_guest(pe_node_t node, pe_working_set_t data_set)
	{
	pe_resource_t *container = node->details->remote_rsc->container;
	pe_action_t *stop = NULL;
	pe_action_t *stonith_op = NULL;

	/* The fence action is just a label; we don't do anything differently for
	* off vs. reboot. We specify it explicitly, rather than let it default to
	* cluster's default action, because we are not _initiating_ fencing -- we
	* are creating a pseudo-event to describe fencing that is already occurring
	* by other means (container recovery).
	*/
	const char *fence_action = "off";

	/* Check whether guest's container resource has any explicit stop or
	* start (the stop may be implied by fencing of the guest's host).
	*/
	if (container) {
	stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);

	if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
	fence_action = "reboot";
	}
	}

	/* Create a fence pseudo-event, so we have an event to order actions
	* against, and the controller can always detect it.
	*/
	stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set);
	update_action_flags(stonith_op, pe_action_pseudo \| pe_action_runnable,
	__FUNCTION__, __LINE__);

	/* We want to imply stops/demotes after the guest is stopped, not wait until
	* it is restarted, so we always order pseudo-fencing after stop, not start
	* (even though start might be closer to what is done for a real reboot).
	*/
	if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
	pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set);
	crm_info("Implying guest node %s is down (action %d) after %s fencing",
	node->details->uname, stonith_op->id, stop->node->details->uname);
	order_actions(parent_stonith_op, stonith_op,
	pe_order_runnable_left\|pe_order_implies_then);

	} else if (stop) {
	order_actions(stop, stonith_op,
	pe_order_runnable_left\|pe_order_implies_then);
	crm_info("Implying guest node %s is down (action %d) "
	"after container %s is stopped (action %d)",
	node->details->uname, stonith_op->id,
	container->id, stop->id);
	} else {
	/* If we're fencing the guest node but there's no stop for the guest
	* resource, we must think the guest is already stopped. However, we may
	* think so because its resource history was just cleaned. To avoid
	* unnecessarily considering the guest node down if it's really up,
	* order the pseudo-fencing after any stop of the connection resource,
	* which will be ordered after any container (re-)probe.
	*/
	stop = find_first_action(node->details->remote_rsc->actions, NULL,
	RSC_STOP, NULL);

	if (stop) {
	order_actions(stop, stonith_op, pe_order_optional);
	crm_info("Implying guest node %s is down (action %d) "
	"after connection is stopped (action %d)",
	node->details->uname, stonith_op->id, stop->id);
	} else {
	/* Not sure why we're fencing, but everything must already be
	* cleanly stopped.
	*/
	crm_info("Implying guest node %s is down (action %d) ",
	node->details->uname, stonith_op->id);
	}
	}

	/* Order/imply other actions relative to pseudo-fence as with real fence */
	pcmk__order_vs_fence(stonith_op, data_set);
	}

	/*
	* Create dependencies for stonith and shutdown operations
	*/
	gboolean
	stage6(pe_working_set_t * data_set)
	{
	pe_action_t *dc_down = NULL;
	pe_action_t *stonith_op = NULL;
	gboolean integrity_lost = FALSE;
	gboolean need_stonith = TRUE;
	GListPtr gIter;
	GListPtr stonith_ops = NULL;
	GList *shutdown_ops = NULL;

	/* Remote ordering constraints need to happen prior to calculating fencing
	* because it is one more place we will mark the node as dirty.
	*
	* A nice side effect of doing them early is that apply_*_ordering() can be
	* simpler because pe_fence_node() has already done some of the work.
	*/
	crm_trace("Creating remote ordering constraints");
	apply_remote_node_ordering(data_set);

	crm_trace("Processing fencing and shutdown cases");
	if (any_managed_resources(data_set) == FALSE) {
	crm_notice("Delaying fencing operations until there are resources to manage");
	need_stonith = FALSE;
	}

	/* Check each node for stonith/shutdown */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	pe_node_t node = (pe_node_t ) gIter->data;

	/* Guest nodes are "fenced" by recovering their container resource,
	* so handle them separately.
	*/
	if (pe__is_guest_node(node)) {
	if (node->details->remote_requires_reset && need_stonith
	&& pe_can_fence(data_set, node)) {
	fence_guest(node, data_set);
	}
	continue;
	}

	stonith_op = NULL;

	if (node->details->unclean
	&& need_stonith && pe_can_fence(data_set, node)) {

	stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
	pe_warn("Scheduling Node %s for STONITH", node->details->uname);

	pcmk__order_vs_fence(stonith_op, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being fenced
	dc_down = stonith_op;

	} else {

	if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
	&& (stonith_ops != NULL)) {
	/* Concurrent fencing is disabled, so order each non-DC
	* fencing in a chain. If there is any DC fencing or
	* shutdown, it will be ordered after the last action in the
	* chain later.
	*/
	order_actions((pe_action_t *) stonith_ops->data,
	stonith_op, pe_order_optional);
	}

	// Remember all non-DC fencing actions in a separate list
	stonith_ops = g_list_prepend(stonith_ops, stonith_op);
	}

	} else if (node->details->online && node->details->shutdown &&
	/* TODO define what a shutdown op means for a remote node.
	* For now we do not send shutdown operations for remote nodes, but
	* if we can come up with a good use for this in the future, we will. */
	pe__is_guest_or_remote_node(node) == FALSE) {

	pe_action_t *down_op = sched_shutdown_op(node, data_set);

	if (node->details->is_dc) {
	// Remember if the DC is being shut down
	dc_down = down_op;
	} else {
	// Remember non-DC shutdowns for later ordering
	shutdown_ops = g_list_prepend(shutdown_ops, down_op);
	}
	}

	if (node->details->unclean && stonith_op == NULL) {
	integrity_lost = TRUE;
	pe_warn("Node %s is unclean!", node->details->uname);
	}
	}

	if (integrity_lost) {
	if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
	pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
	pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");

	} else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
	crm_notice("Cannot fence unclean nodes until quorum is"
	" attained (or no-quorum-policy is set to ignore)");
	}
	}

	if (dc_down != NULL) {
	/* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
	* DC elections. However, we don't want to order non-DC shutdowns before
	* a DC fencing, because even though we don't want a node that's
	* shutting down to become DC, the DC fencing could be ordered before a
	* clone stop that's also ordered before the shutdowns, thus leading to
	* a graph loop.
	*/
	if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
	pe_action_t node_stop = (pe_action_t ) gIter->data;

	crm_debug("Ordering shutdown on %s before %s on DC %s",
	node_stop->node->details->uname,
	dc_down->task, dc_down->node->details->uname);

	order_actions(node_stop, dc_down, pe_order_optional);
	}
	}

	// Order any non-DC fencing before any DC fencing or shutdown

	if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
	/* With concurrent fencing, order each non-DC fencing action
	* separately before any DC fencing or shutdown.
	*/
	for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
	order_actions((pe_action_t *) gIter->data, dc_down,
	pe_order_optional);
	}
	} else if (stonith_ops) {
	/* Without concurrent fencing, the non-DC fencing actions are
	* already ordered relative to each other, so we just need to order
	* the DC fencing after the last action in the chain (which is the
	* first item in the list).
	*/
	order_actions((pe_action_t *) stonith_ops->data, dc_down,
	pe_order_optional);
	}
	}
	g_list_free(stonith_ops);
	g_list_free(shutdown_ops);
	return TRUE;
	}

	/*
	* Determine the sets of independent actions and the correct order for the
	* actions in each set.
	*
	* Mark dependencies of un-runnable actions un-runnable
	*
	*/
	static GListPtr
	find_actions_by_task(GListPtr actions, pe_resource_t * rsc, const char *original_key)
	{
	GListPtr list = NULL;

	list = find_actions(actions, original_key, NULL);
	if (list == NULL) {
	/* we're potentially searching a child of the original resource */
	char *key = NULL;
	char *task = NULL;
	guint interval_ms = 0;

	if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
	key = pcmk__op_key(rsc->id, task, interval_ms);
	list = find_actions(actions, key, NULL);

	} else {
	crm_err("search key: %s", original_key);
	}

	free(key);
	free(task);
	}

	return list;
	}

	static void
	rsc_order_then(pe_action_t lh_action, pe_resource_t rsc,
	pe__ordering_t *order)
	{
	GListPtr gIter = NULL;
	GListPtr rh_actions = NULL;
	pe_action_t *rh_action = NULL;
	enum pe_ordering type;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(order != NULL, return);

	type = order->type;
	rh_action = order->rh_action;
	crm_trace("Processing RH of ordering constraint %d", order->id);

	if (rh_action != NULL) {
	rh_actions = g_list_prepend(NULL, rh_action);

	} else if (rsc != NULL) {
	rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
	}

	if (rh_actions == NULL) {
	pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
	" ignoring", rsc->id, order->rh_action_task);
	if (lh_action) {
	pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
	}
	return;
	}

	if ((lh_action != NULL) && (lh_action->rsc == rsc)
	&& pcmk_is_set(lh_action->flags, pe_action_dangle)) {

	pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
	order->rh_action_task);
	pe__clear_order_flags(type, pe_order_implies_then);
	}

	gIter = rh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) gIter->data;

	if (lh_action) {
	order_actions(lh_action, rh_action_iter, type);

	} else if (type & pe_order_implies_then) {
	update_action_flags(rh_action_iter, pe_action_runnable \| pe_action_clear, __FUNCTION__, __LINE__);
	crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
	} else {
	crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
	}
	}

	g_list_free(rh_actions);
	}

	static void
	rsc_order_first(pe_resource_t lh_rsc, pe__ordering_t order,
	pe_working_set_t *data_set)
	{
	GListPtr gIter = NULL;
	GListPtr lh_actions = NULL;
	pe_action_t *lh_action = order->lh_action;
	pe_resource_t *rh_rsc = order->rh_rsc;

	crm_trace("Processing LH of ordering constraint %d", order->id);
	CRM_ASSERT(lh_rsc != NULL);

	if (lh_action != NULL) {
	lh_actions = g_list_prepend(NULL, lh_action);

	} else {
	lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
	}

	if (lh_actions == NULL && lh_rsc != rh_rsc) {
	char *key = NULL;
	char *op_type = NULL;
	guint interval_ms = 0;

	parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
	key = pcmk__op_key(lh_rsc->id, op_type, interval_ms);

	if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else {
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
	lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
	lh_actions = g_list_prepend(NULL, lh_action);
	}

	free(op_type);
	}

	gIter = lh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t lh_action_iter = (pe_action_t ) gIter->data;

	if (rh_rsc == NULL && order->rh_action) {
	rh_rsc = order->rh_action->rsc;
	}
	if (rh_rsc) {
	rsc_order_then(lh_action_iter, rh_rsc, order);

	} else if (order->rh_action) {
	order_actions(lh_action_iter, order->rh_action, order->type);
	}
	}

	g_list_free(lh_actions);
	}

	extern void update_colo_start_chain(pe_action_t *action,
	pe_working_set_t *data_set);

	static int
	is_recurring_action(pe_action_t *action)
	{
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	return 0;
	}
	return (interval_ms > 0);
	}

	static void
	apply_container_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	/* VMs are also classified as containers for these purposes... in
	* that they both involve a 'thing' running on a real or remote
	* cluster node.
	*
	* This allows us to be smarter about the type and extent of
	* recovery actions required in various scenarios
	*/
	pe_resource_t *remote_rsc = NULL;
	pe_resource_t *container = NULL;
	enum action_tasks task = text2task(action->task);

	CRM_ASSERT(action->rsc);
	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	container = remote_rsc->container;
	CRM_ASSERT(container);

	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	pe_fence_node(data_set, action->node, "container failed", FALSE);
	}

	crm_trace("Order %s action %s relative to %s%s for %s%s",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id,
	pcmk_is_set(container->flags, pe_rsc_failed)? "failed " : "",
	container->id);

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	/* Force resource recovery if the container is recovered */
	order_start_then_action(container, action, pe_order_implies_then,
	data_set);

	/* Wait for the connection resource to be up too */
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	break;

	case stop_rsc:
	case action_demote:
	if (pcmk_is_set(container->flags, pe_rsc_failed)) {
	/* When the container representing a guest node fails, any stop
	* or demote actions for resources running on the guest node
	* are implied by the container stopping. This is similar to
	* how fencing operations work for cluster nodes and remote
	* nodes.
	*/
	} else {
	/* Ensure the operation happens before the connection is brought
	* down.
	*
	* If we really wanted to, we could order these after the
	* connection start, IFF the container's current role was
	* stopped (otherwise we re-introduce an ordering loop when the
	* connection is restarting).
	*/
	order_action_then_stop(action, remote_rsc, pe_order_none,
	data_set);
	}
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	if(task != no_action) {
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);
	}
	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	break;
	}
	}

	static enum remote_connection_state
	get_remote_node_state(pe_node_t *node)
	{
	pe_resource_t *remote_rsc = NULL;
	pe_node_t *cluster_node = NULL;

	CRM_ASSERT(node);

	remote_rsc = node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	cluster_node = pe__current_node(remote_rsc);

	/* If the cluster node the remote connection resource resides on
	* is unclean or went offline, we can't process any operations
	* on that remote node until after it starts elsewhere.
	*/
	if(remote_rsc->next_role == RSC_ROLE_STOPPED \|\| remote_rsc->allocated_to == NULL) {
	/* The connection resource is not going to run anywhere */

	if (cluster_node && cluster_node->details->unclean) {
	/* The remote connection is failed because its resource is on a
	* failed node and can't be recovered elsewhere, so we must fence.
	*/
	return remote_state_failed;
	}

	if (!pcmk_is_set(remote_rsc->flags, pe_rsc_failed)) {
	/* Connection resource is cleanly stopped */
	return remote_state_stopped;
	}

	/* Connection resource is failed */

	if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
	&& remote_rsc->remote_reconnect_ms
	&& node->details->remote_was_fenced
	&& !pe__shutdown_requested(node)) {

	/* We won't know whether the connection is recoverable until the
	* reconnect interval expires and we reattempt connection.
	*/
	return remote_state_unknown;
	}

	/* The remote connection is in a failed state. If there are any
	* resources known to be active on it (stop) or in an unknown state
	* (probe), we must assume the worst and fence it.
	*/
	return remote_state_failed;

	} else if (cluster_node == NULL) {
	/* Connection is recoverable but not currently running anywhere, see if we can recover it first */
	return remote_state_unknown;

	} else if(cluster_node->details->unclean == TRUE
	\|\| cluster_node->details->online == FALSE) {
	/* Connection is running on a dead node, see if we can recover it first */
	return remote_state_resting;

	} else if (pcmk__list_of_multiple(remote_rsc->running_on)
	&& remote_rsc->partial_migration_source
	&& remote_rsc->partial_migration_target) {
	/* We're in the middle of migrating a connection resource,
	* wait until after the resource migrates before performing
	* any actions.
	*/
	return remote_state_resting;

	}
	return remote_state_alive;
	}

	/*!
	* \internal
	* \brief Order actions on remote node relative to actions for the connection
	*/
	static void
	apply_remote_ordering(pe_action_t action, pe_working_set_t data_set)
	{
	pe_resource_t *remote_rsc = NULL;
	enum action_tasks task = text2task(action->task);
	enum remote_connection_state state = get_remote_node_state(action->node);

	enum pe_ordering order_opts = pe_order_none;

	if (action->rsc == NULL) {
	return;
	}

	CRM_ASSERT(action->node);
	CRM_ASSERT(pe__is_guest_or_remote_node(action->node));

	remote_rsc = action->node->details->remote_rsc;
	CRM_ASSERT(remote_rsc);

	crm_trace("Order %s action %s relative to %s%s (state: %s)",
	action->task, action->uuid,
	pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
	remote_rsc->id, state2text(state));

	if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
	/* Migration ops map to "no_action", but we need to apply the same
	* ordering as for stop or demote (see get_router_node()).
	*/
	task = stop_rsc;
	}

	switch (task) {
	case start_rsc:
	case action_promote:
	order_opts = pe_order_none;

	if (state == remote_state_failed) {
	/* Force recovery, by making this action required */
	pe__set_order_flags(order_opts, pe_order_implies_then);
	}

	/* Ensure connection is up before running this action */
	order_start_then_action(remote_rsc, action, order_opts, data_set);
	break;

	case stop_rsc:
	if(state == remote_state_alive) {
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else if(state == remote_state_failed) {
	/* The resource is active on the node, but since we don't have a
	* valid connection, the only way to stop the resource is by
	* fencing the node. There is no need to order the stop relative
	* to the remote connection, since the stop will become implied
	* by the fencing.
	*/
	pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE);

	} else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
	/* State must be remote_state_unknown or remote_state_stopped.
	* Since the connection is not coming back up in this
	* transition, stop this resource first.
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_implies_first, data_set);

	} else {
	/* The connection is going to be started somewhere else, so
	* stop this resource after that completes.
	*/
	order_start_then_action(remote_rsc, action, pe_order_none, data_set);
	}
	break;

	case action_demote:
	/* Only order this demote relative to the connection start if the
	* connection isn't being torn down. Otherwise, the demote would be
	* blocked because the connection start would not be allowed.
	*/
	if(state == remote_state_resting \|\| state == remote_state_unknown) {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	} /* Otherwise we can rely on the stop ordering */
	break;

	default:
	/* Wait for the connection resource to be up */
	if (is_recurring_action(action)) {
	/* In case we ever get the recovery logic wrong, force
	* recurring monitors to be restarted, even if just
	* the connection was re-established
	*/
	order_start_then_action(remote_rsc, action,
	pe_order_implies_then, data_set);

	} else {
	pe_node_t *cluster_node = pe__current_node(remote_rsc);

	if(task == monitor_rsc && state == remote_state_failed) {
	/* We would only be here if we do not know the
	* state of the resource on the remote node.
	* Since we have no way to find out, it is
	* necessary to fence the node.
	*/
	pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE);
	}

	if(cluster_node && state == remote_state_stopped) {
	/* The connection is currently up, but is going
	* down permanently.
	*
	* Make sure we check services are actually
	* stopped _before_ we let the connection get
	* closed
	*/
	order_action_then_stop(action, remote_rsc,
	pe_order_runnable_left, data_set);

	} else {
	order_start_then_action(remote_rsc, action, pe_order_none,
	data_set);
	}
	}
	break;
	}
	}

	static void
	apply_remote_node_ordering(pe_working_set_t *data_set)
	{
	if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	return;
	}

	for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_resource_t *remote = NULL;

	// We are only interested in resource actions
	if (action->rsc == NULL) {
	continue;
	}

	/* Special case: If we are clearing the failcount of an actual
	* remote connection resource, then make sure this happens before
	* any start of the resource in this transition.
	*/
	if (action->rsc->is_remote_node &&
	pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {

	custom_action_order(action->rsc,
	NULL,
	action,
	action->rsc,
	pcmk__op_key(action->rsc->id, RSC_START, 0),
	NULL,
	pe_order_optional,
	data_set);

	continue;
	}

	// We are only interested in actions allocated to a node
	if (action->node == NULL) {
	continue;
	}

	if (!pe__is_guest_or_remote_node(action->node)) {
	continue;
	}

	/* We are only interested in real actions.
	*
	* @TODO This is probably wrong; pseudo-actions might be converted to
	* real actions and vice versa later in update_actions() at the end of
	* stage7().
	*/
	if (pcmk_is_set(action->flags, pe_action_pseudo)) {
	continue;
	}

	remote = action->node->details->remote_rsc;
	if (remote == NULL) {
	// Orphaned
	continue;
	}

	/* Another special case: if a resource is moving to a Pacemaker Remote
	* node, order the stop on the original node after any start of the
	* remote connection. This ensures that if the connection fails to
	* start, we leave the resource running on the original node.
	*/
	if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
	for (GList *item = action->rsc->actions; item != NULL;
	item = item->next) {
	pe_action_t *rsc_action = item->data;

	if ((rsc_action->node->details != action->node->details)
	&& pcmk__str_eq(rsc_action->task, RSC_STOP, pcmk__str_casei)) {
	custom_action_order(remote, start_key(remote), NULL,
	action->rsc, NULL, rsc_action,
	pe_order_optional, data_set);
	}
	}
	}

	/* The action occurs across a remote connection, so create
	* ordering constraints that guarantee the action occurs while the node
	* is active (after start, before stop ... things like that).
	*
	* This is somewhat brittle in that we need to make sure the results of
	* this ordering are compatible with the result of get_router_node().
	* It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
	* of this logic rather than action2xml().
	*/
	if (remote->container) {
	crm_trace("Container ordering for %s", action->uuid);
	apply_container_ordering(action, data_set);

	} else {
	crm_trace("Remote ordering for %s", action->uuid);
	apply_remote_ordering(action, data_set);
	}
	}
	}

	static gboolean
	order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
	{
	/* No need to probe the resource on the node that is being
	* unfenced. Otherwise it might introduce transition loop
	* since probe will be performed after the node is
	* unfenced.
	*/
	if (pcmk__str_eq(rh_action->task, CRM_OP_FENCE, pcmk__str_casei)
	&& probe->node && rh_action->node
	&& probe->node->details == rh_action->node->details) {
	const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");

	if (pcmk__str_eq(op, "on", pcmk__str_casei)) {
	return TRUE;
	}
	}

	// Shutdown waits for probe to complete only if it's on the same node
	if ((pcmk__str_eq(rh_action->task, CRM_OP_SHUTDOWN, pcmk__str_casei))
	&& probe->node && rh_action->node
	&& probe->node->details != rh_action->node->details) {
	return TRUE;
	}
	return FALSE;
	}

	static void
	order_first_probes_imply_stops(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	enum pe_ordering order_type = pe_order_optional;

	pe_resource_t *lh_rsc = order->lh_rsc;
	pe_resource_t *rh_rsc = order->rh_rsc;
	pe_action_t *lh_action = order->lh_action;
	pe_action_t *rh_action = order->rh_action;
	const char *lh_action_task = order->lh_action_task;
	const char *rh_action_task = order->rh_action_task;

	GListPtr probes = NULL;
	GListPtr rh_actions = NULL;

	GListPtr pIter = NULL;

	if (lh_rsc == NULL) {
	continue;

	} else if (rh_rsc && lh_rsc == rh_rsc) {
	continue;
	}

	if (lh_action == NULL && lh_action_task == NULL) {
	continue;
	}

	if (rh_action == NULL && rh_action_task == NULL) {
	continue;
	}

	/* Technically probe is expected to return "not running", which could be
	* the alternative of stop action if the status of the resource is
	* unknown yet.
	*/
	if (lh_action && !pcmk__str_eq(lh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (lh_action == NULL
	&& lh_action_task
	&& !pcmk__ends_with(lh_action_task, "_" RSC_STOP "_0")) {
	continue;
	}

	/* Do not probe the resource inside of a stopping container. Otherwise
	* it might introduce transition loop since probe will be performed
	* after the container starts again.
	*/
	if (rh_rsc && lh_rsc->container == rh_rsc) {
	if (rh_action && pcmk__str_eq(rh_action->task, RSC_STOP, pcmk__str_casei)) {
	continue;

	} else if (rh_action == NULL && rh_action_task
	&& pcmk__ends_with(rh_action_task,"_" RSC_STOP "_0")) {
	continue;
	}
	}

	if (order->type == pe_order_none) {
	continue;
	}

	// Preserve the order options for future filtering
	if (pcmk_is_set(order->type, pe_order_apply_first_non_migratable)) {
	pe__set_order_flags(order_type,
	pe_order_apply_first_non_migratable);
	}

	if (pcmk_is_set(order->type, pe_order_same_node)) {
	pe__set_order_flags(order_type, pe_order_same_node);
	}

	// Keep the order types for future filtering
	if (order->type == pe_order_anti_colocation
	\|\| order->type == pe_order_load) {
	order_type = order->type;
	}

	probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
	if (probes == NULL) {
	continue;
	}

	if (rh_action) {
	rh_actions = g_list_prepend(rh_actions, rh_action);

	} else if (rh_rsc && rh_action_task) {
	rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
	}

	if (rh_actions == NULL) {
	g_list_free(probes);
	continue;
	}

	crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
	" (id=%d, type=%.6x)",
	lh_action ? lh_action->uuid : lh_action_task,
	rh_action ? rh_action->uuid : rh_action_task,
	order->id, order->type);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;
	GListPtr rIter = NULL;

	for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
	pe_action_t rh_action_iter = (pe_action_t ) rIter->data;

	if (order_first_probe_unneeded(probe, rh_action_iter)) {
	continue;
	}
	order_actions(probe, rh_action_iter, order_type);
	}
	}

	g_list_free(rh_actions);
	g_list_free(probes);
	}
	}

	static void
	order_first_probe_then_restart_repromote(pe_action_t * probe,
	pe_action_t * after,
	pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	bool interleave = FALSE;
	pe_resource_t *compatible_rsc = NULL;

	if (probe == NULL
	\|\| probe->rsc == NULL
	\|\| probe->rsc->variant != pe_native) {
	return;
	}

	if (after == NULL
	// Avoid running into any possible loop
	\|\| pcmk_is_set(after->flags, pe_action_tracking)) {
	return;
	}

	if (!pcmk__str_eq(probe->task, RSC_STATUS, pcmk__str_casei)) {
	return;
	}

	pe__set_action_flags(after, pe_action_tracking);

	crm_trace("Processing based on %s %s -> %s %s",
	probe->uuid,
	probe->node ? probe->node->details->uname: "",
	after->uuid,
	after->node ? after->node->details->uname : "");

	if (after->rsc
	/* Better not build a dependency directly with a clone/group.
	* We are going to proceed through the ordering chain and build
	* dependencies with its children.
	*/
	&& after->rsc->variant == pe_native
	&& probe->rsc != after->rsc) {

	GListPtr then_actions = NULL;
	enum pe_ordering probe_order_type = pe_order_optional;

	if (pcmk__str_eq(after->task, RSC_START, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, RSC_PROMOTE, pcmk__str_casei)) {
	then_actions = pe__resource_actions(after->rsc, NULL, RSC_DEMOTE, FALSE);
	}

	for (gIter = then_actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t then = (pe_action_t ) gIter->data;

	// Skip any pseudo action which for example is implied by fencing
	if (pcmk_is_set(then->flags, pe_action_pseudo)) {
	continue;
	}

	order_actions(probe, then, probe_order_type);
	}
	g_list_free(then_actions);
	}

	if (after->rsc
	&& after->rsc->variant > pe_group) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	XML_RSC_ATTR_INTERLEAVE);

	interleave = crm_is_true(interleave_s);

	if (interleave) {
	/* For an interleaved clone, we should build a dependency only
	* with the relevant clone child.
	*/
	compatible_rsc = find_compatible_child(probe->rsc,
	after->rsc,
	RSC_ROLE_UNKNOWN,
	FALSE, data_set);
	}
	}

	for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) gIter->data;
	/* pe_order_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pe_order_implies_then_on_node though. It's now only
	* used for unfencing case, which tends to introduce transition
	* loops...
	*/

	if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pe_order_implies_first_printed \| pe_order_runnable_left
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if (after->rsc == NULL
	\|\| after->rsc->variant < pe_group
	\|\| probe->rsc->parent == after->rsc
	\|\| after_wrapper->action->rsc == NULL
	\|\| after_wrapper->action->rsc->variant > pe_group
	\|\| after->rsc != after_wrapper->action->rsc->parent) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if (after->rsc->variant > pe_group
	&& interleave == TRUE
	&& (compatible_rsc == NULL
	\|\| compatible_rsc != after_wrapper->action->rsc)) {
	continue;
	}
	}

	crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)",
	after->uuid,
	after->node ? after->node->details->uname: "",
	after_wrapper->action->uuid,
	after_wrapper->action->node ? after_wrapper->action->node->details->uname : "",
	after_wrapper->type);

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	}
	}

	static void clear_actions_tracking_flag(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (pcmk_is_set(action->flags, pe_action_tracking)) {
	pe__clear_action_flags(action, pe_action_tracking);
	}
	}
	}

	static void
	order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	GListPtr probes = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pe_resource_t * child = (pe_resource_t *) gIter->data;

	order_first_rsc_probes(child, data_set);
	}

	if (rsc->variant != pe_native) {
	return;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (gIter = probes; gIter != NULL; gIter= gIter->next) {
	pe_action_t probe = (pe_action_t ) gIter->data;
	GListPtr aIter = NULL;

	for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) {
	pe_action_wrapper_t after_wrapper = (pe_action_wrapper_t ) aIter->data;

	order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
	clear_actions_tracking_flag(data_set);
	}
	}

	g_list_free(probes);
	}

	static void
	order_first_probes(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	order_first_rsc_probes(rsc, data_set);
	}

	order_first_probes_imply_stops(data_set);
	}

	static void
	order_then_probes(pe_working_set_t * data_set)
	{
	#if 0
	GListPtr gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	/* Given "A then B", we would prefer to wait for A to be
	* started before probing B.
	*
	* If A was a filesystem on which the binaries and data for B
	* lived, it would have been useful if the author of B's agent
	* could assume that A is running before B.monitor will be
	* called.
	*
	* However we can't _only_ probe once A is running, otherwise
	* we'd not detect the state of B if A could not be started
	* for some reason.
	*
	* In practice however, we cannot even do an opportunistic
	* version of this because B may be moving:
	*
	* B.probe -> B.start
	* B.probe -> B.stop
	* B.stop -> B.start
	* A.stop -> A.start
	* A.start -> B.probe
	*
	* So far so good, but if we add the result of this code:
	*
	* B.stop -> A.stop
	*
	* Then we get a loop:
	*
	* B.probe -> B.stop -> A.stop -> A.start -> B.probe
	*
	* We could kill the 'B.probe -> B.stop' dependency, but that
	* could mean stopping B "too" soon, because B.start must wait
	* for the probes to complete.
	*
	* Another option is to allow it only if A is a non-unique
	* clone with clone-max == node-max (since we'll never be
	* moving it). However, we could still be stopping one
	* instance at the same time as starting another.

	* The complexity of checking for allowed conditions combined
	* with the ever narrowing usecase suggests that this code
	* should remain disabled until someone gets smarter.
	*/
	pe_action_t *start = NULL;
	GListPtr actions = NULL;
	GListPtr probes = NULL;

	actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if(start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);

	for (actions = start->actions_before; actions != NULL; actions = actions->next) {
	pe_action_wrapper_t before = (pe_action_wrapper_t ) actions->data;

	GListPtr pIter = NULL;
	pe_action_t *first = before->action;
	pe_resource_t *first_rsc = first->rsc;

	if(first->required_runnable_before) {
	GListPtr clone_actions = NULL;
	for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
	before = (pe_action_wrapper_t *) clone_actions->data;

	crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);

	CRM_ASSERT(before->action->rsc);
	first_rsc = before->action->rsc;
	break;
	}

	} else if(!pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if(first_rsc == NULL) {
	continue;

	} else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	pe_action_t probe = (pe_action_t ) pIter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pe_order_optional);
	}
	}
	}
	#endif
	}

	static void
	order_probes(pe_working_set_t * data_set)
	{
	order_first_probes(data_set);
	order_then_probes(data_set);
	}

	gboolean
	stage7(pe_working_set_t * data_set)
	{
	GList *gIter = NULL;

	crm_trace("Applying ordering constraints");

	/* Don't ask me why, but apparently they need to be processed in
	* the order they were created in... go figure
	*
	* Also g_list_append() has horrendous performance characteristics
	* So we need to use g_list_prepend() and then reverse the list here
	*/
	data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	pe__ordering_t *order = gIter->data;
	pe_resource_t *rsc = order->lh_rsc;

	crm_trace("Applying ordering constraint: %d", order->id);

	if (rsc != NULL) {
	crm_trace("rsc_action-to-*");
	rsc_order_first(rsc, order, data_set);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	crm_trace("action-to-rsc_action");
	rsc_order_then(order->lh_action, rsc, order);

	} else {
	crm_trace("action-to-action");
	order_actions(order->lh_action, order->rh_action, order->type);
	}
	}

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	update_colo_start_chain(action, data_set);
	}

	crm_trace("Ordering probes");
	order_probes(data_set);

	crm_trace("Updating %d actions", g_list_length(data_set->actions));
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	update_action(action, data_set);
	}

	// Check for invalid orderings
	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;
	pe_action_wrapper_t *input = NULL;

	for (GList *input_iter = action->actions_before;
	input_iter != NULL; input_iter = input_iter->next) {

	input = (pe_action_wrapper_t *) input_iter->data;
	if (pcmk__ordering_is_invalid(action, input)) {
	input->type = pe_order_none;
	}
	}
	}

	LogNodeActions(data_set, FALSE);
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	LogActions(rsc, data_set, FALSE);
	}
	return TRUE;
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error) {
	crm_err("Calculated transition %d (with errors), saving inputs in %s",
	transition_id, filename);

	} else if (was_processing_warning) {
	crm_warn("Calculated transition %d (with warnings), saving inputs in %s",
	transition_id, filename);

	} else {
	crm_notice("Calculated transition %d, saving inputs in %s",
	transition_id, filename);
	}
	- if (crm_config_error) {
	+ if (pcmk__config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*
	* Create a dependency graph to send to the transitioner (via the controller)
	*/
	gboolean
	stage8(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	const char *value = NULL;

	transition_id++;
	crm_trace("Creating transition graph %d.", transition_id);

	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(data_set->config_hash, "cluster-delay");
	crm_xml_add(data_set->graph, "cluster-delay", value);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	crm_xml_add(data_set->graph, "stonith-timeout", value);

	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
	crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(data_set->graph, "failed-start-offset", "1");
	}

	value = pe_pref(data_set->config_hash, "batch-limit");
	crm_xml_add(data_set->graph, "batch-limit", value);

	crm_xml_add_int(data_set->graph, "transition_id", transition_id);

	value = pe_pref(data_set->config_hash, "migration-limit");
	if (crm_parse_ll(value, NULL) > 0) {
	crm_xml_add(data_set->graph, "migration-limit", value);
	}

	if (data_set->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) data_set->recheck_by);
	crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* errors...
	slist_iter(action, pe_action_t, action_list, lpc,
	if(action->optional == FALSE && action->runnable == FALSE) {
	print_action("Ignoring", action, TRUE);
	}
	);
	*/

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	gIter = data_set->resources;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_resource_t rsc = (pe_resource_t ) gIter->data;

	pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
	rsc->cmds->expand(rsc, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created resource-driven action list");

	/* pseudo action to distribute list of nodes with maintenance state update */
	add_maintenance_update(data_set);

	/* catch any non-resource specific actions */
	crm_trace("processing non-resource actions");

	gIter = data_set->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc
	&& action->node
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pe_action_optional\|pe_action_runnable)
	&& pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)
	) {
	/* Eventually we should just ignore the 'fence' case
	* But for now it's the best way to detect (in CTS) when
	* CIB resource updates are being lost
	*/
	if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
	\|\| data_set->no_quorum_policy == no_quorum_ignore) {
	crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
	action->node->details->unclean ? "fence" : "shut down",
	action->node->details->uname, action->rsc->id,
	pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
	pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
	action->uuid);
	}
	}

	graph_element_from_action(action, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created generic action list");
	crm_trace("Created transition graph %d.", transition_id);

	return TRUE;
	}

	void
	LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
	{
	GListPtr gIter = NULL;

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pe_action_t action = (pe_action_t ) gIter->data;

	if (action->rsc != NULL) {
	continue;
	} else if (pcmk_is_set(action->flags, pe_action_optional)) {
	continue;
	}

	if (pe__is_guest_node(action->node)) {
	node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
	} else if(action->node) {
	node_name = crm_strdup_printf("%s", action->node->details->uname);
	}


	if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
	task = strdup("Shutdown");
	} else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");
	task = crm_strdup_printf("Fence (%s)", op);
	}

	if(task == NULL) {
	/* Nothing to report */
	} else if(terminal && action->reason) {
	printf(" * %s %s '%s'\n", task, node_name, action->reason);
	} else if(terminal) {
	printf(" * %s %s\n", task, node_name);
	} else if(action->reason) {
	crm_notice(" * %s %s '%s'\n", task, node_name, action->reason);
	} else {
	crm_notice(" * %s %s\n", task, node_name);
	}

	free(node_name);
	free(task);
	}
	}
	diff --git a/tools/crm_verify.c b/tools/crm_verify.c
	index e6ab9aa75e..1df27ec5a7 100644
	--- a/tools/crm_verify.c
	+++ b/tools/crm_verify.c
	@@ -1,318 +1,318 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <glib.h>

	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/msg_xml.h>
	#include <crm/cib.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	gboolean USE_LIVE_CIB = FALSE;
	char *cib_save = NULL;
	extern gboolean stage0(pe_working_set_t * data_set);

	static pcmk__cli_option_t long_options[] = {
	// long option, argument type, storage, short option, description, flags
	{
	"help", no_argument, NULL, '?',
	"\tThis text", pcmk__option_default
	},
	{
	"version", no_argument, NULL, '$',
	"\tVersion information", pcmk__option_default
	},
	{
	"verbose", no_argument, NULL, 'V',
	"\tSpecify multiple times to increase debug output\n",
	pcmk__option_default
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"\nData sources:", pcmk__option_default
	},
	{
	"live-check", no_argument, NULL, 'L',
	"Check the configuration used by the running cluster\n",
	pcmk__option_default
	},
	{
	"xml-file", required_argument, NULL, 'x',
	"Check the configuration in the named file", pcmk__option_default
	},
	{
	"xml-text", required_argument, NULL, 'X',
	"Check the configuration in the supplied string", pcmk__option_default
	},
	{
	"xml-pipe", no_argument, NULL, 'p',
	"Check the configuration piped in via stdin", pcmk__option_default
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"\nAdditional Options:", pcmk__option_default
	},
	{
	"save-xml", required_argument, 0, 'S',
	"Save verified XML to named file (most useful with -L)",
	pcmk__option_default
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"\nExamples:", pcmk__option_paragraph
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"Check the consistency of the configuration in the running cluster:",
	pcmk__option_paragraph
	},
	{
	"-spacer-", no_argument, NULL, '-',
	" crm_verify --live-check", pcmk__option_example
	},
	{
	"-spacer-", no_argument, NULL, '-',
	"Check the consistency of the configuration in a given file and "
	"produce verbose output:",
	pcmk__option_paragraph
	},
	{
	"-spacer-", no_argument, NULL, '-',
	" crm_verify --xml-file file.xml --verbose", pcmk__option_example
	},
	{0, 0, 0, 0}
	};

	int
	main(int argc, char **argv)
	{
	xmlNode *cib_object = NULL;
	xmlNode *status = NULL;
	int argerr = 0;
	int flag;
	int option_index = 0;

	pe_working_set_t *data_set = NULL;
	cib_t *cib_conn = NULL;
	int rc = pcmk_ok;

	bool verbose = FALSE;
	gboolean xml_stdin = FALSE;
	const char *xml_tag = NULL;
	const char *xml_file = NULL;
	const char *xml_string = NULL;

	crm_log_cli_init("crm_verify");
	pcmk__set_cli_options(NULL, "[options]", long_options,
	"check a Pacemaker configuration for errors\n\n"
	"Check the well-formedness of a complete Pacemaker "
	"XML configuration,\nits conformance to the "
	"configured schema, and the presence of common\n"
	"misconfigurations. Problems reported as errors "
	"must be fixed before the\ncluster will work "
	"properly. It is left to the administrator to decide"
	"\nwhether to fix problems reported as warnings.");

	while (1) {
	flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
	if (flag == -1)
	break;

	switch (flag) {
	case 'X':
	crm_trace("Option %c => %s", flag, optarg);
	xml_string = optarg;
	break;
	case 'x':
	crm_trace("Option %c => %s", flag, optarg);
	xml_file = optarg;
	break;
	case 'p':
	xml_stdin = TRUE;
	break;
	case 'S':
	cib_save = optarg;
	break;
	case 'V':
	verbose = TRUE;
	crm_bump_log_level(argc, argv);
	break;
	case 'L':
	USE_LIVE_CIB = TRUE;
	break;
	case '$':
	case '?':
	pcmk__cli_help(flag, CRM_EX_OK);
	break;
	default:
	fprintf(stderr, "Option -%c is not yet supported\n", flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc) {
	printf("%s ", argv[optind++]);
	}
	printf("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_err("%d errors in option parsing", argerr);
	pcmk__cli_help(flag, CRM_EX_USAGE);
	}

	crm_info("=#=#=#=#= Getting XML =#=#=#=#=");

	if (USE_LIVE_CIB) {
	cib_conn = cib_new();
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	}

	if (USE_LIVE_CIB) {
	if (rc == pcmk_ok) {
	int options = cib_scope_local \| cib_sync_call;

	crm_info("Reading XML from: live cluster");
	rc = cib_conn->cmds->query(cib_conn, NULL, &cib_object, options);
	}

	if (rc != pcmk_ok) {
	fprintf(stderr, "Live CIB query failed: %s\n", pcmk_strerror(rc));
	goto done;
	}
	if (cib_object == NULL) {
	fprintf(stderr, "Live CIB query failed: empty result\n");
	rc = -ENOMSG;
	goto done;
	}

	} else if (xml_file != NULL) {
	cib_object = filename2xml(xml_file);
	if (cib_object == NULL) {
	fprintf(stderr, "Couldn't parse input file: %s\n", xml_file);
	rc = -ENODATA;
	goto done;
	}

	} else if (xml_string != NULL) {
	cib_object = string2xml(xml_string);
	if (cib_object == NULL) {
	fprintf(stderr, "Couldn't parse input string: %s\n", xml_string);
	rc = -ENODATA;
	goto done;
	}
	} else if (xml_stdin) {
	cib_object = stdin2xml();
	if (cib_object == NULL) {
	fprintf(stderr, "Couldn't parse input from STDIN.\n");
	rc = -ENODATA;
	goto done;
	}

	} else {
	fprintf(stderr, "No configuration source specified."
	" Use --help for usage information.\n");
	rc = -ENODATA;
	goto done;
	}

	xml_tag = crm_element_name(cib_object);
	if (!pcmk__str_eq(xml_tag, XML_TAG_CIB, pcmk__str_casei)) {
	fprintf(stderr,
	"This tool can only check complete configurations (i.e. those starting with <cib>).\n");
	rc = -EBADMSG;
	goto done;
	}

	if (cib_save != NULL) {
	write_xml_file(cib_object, cib_save, FALSE);
	}

	status = get_object_root(XML_CIB_TAG_STATUS, cib_object);
	if (status == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (validate_xml(cib_object, NULL, FALSE) == FALSE) {
	pcmk__config_err("CIB did not pass schema validation");
	free_xml(cib_object);
	cib_object = NULL;

	} else if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	- crm_config_error = TRUE;
	+ pcmk__config_error = true;
	free_xml(cib_object);
	cib_object = NULL;
	fprintf(stderr, "The cluster will NOT be able to use this configuration.\n");
	fprintf(stderr, "Please manually update the configuration to conform to the %s syntax.\n",
	xml_latest_schema());
	}

	data_set = pe_new_working_set();
	if (data_set == NULL) {
	rc = -errno;
	crm_perror(LOG_CRIT, "Unable to allocate working set");
	goto done;
	}
	pe__set_working_set_flags(data_set, pe_flag_no_counts\|pe_flag_no_compat);

	if (cib_object == NULL) {
	} else if (status != NULL \|\| USE_LIVE_CIB) {
	/* live queries will always have a status section and can do a full simulation */
	pcmk__schedule_actions(data_set, cib_object, NULL);

	} else {
	data_set->now = crm_time_new(NULL);
	data_set->input = cib_object;
	stage0(data_set);
	}
	pe_free_working_set(data_set);

	- if (crm_config_error) {
	+ if (pcmk__config_error) {
	fprintf(stderr, "Errors found during check: config not valid\n");
	if (verbose == FALSE) {
	fprintf(stderr, " -V may provide more details\n");
	}
	rc = -pcmk_err_schema_validation;

	- } else if (crm_config_warning) {
	+ } else if (pcmk__config_warning) {
	fprintf(stderr, "Warnings found during check: config may not be valid\n");
	if (verbose == FALSE) {
	fprintf(stderr, " Use -V -V for more details\n");
	}
	rc = -pcmk_err_schema_validation;
	}

	if (USE_LIVE_CIB && cib_conn) {
	cib_conn->cmds->signoff(cib_conn);
	cib_delete(cib_conn);
	}

	done:
	crm_exit(crm_errno2exit(rc));
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Jun 25, 4:10 AM (1 d, 10 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1952101
Default Alt Text: (206 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions