No OneTemporary
Actions

Size

217 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/based/cibmon.c b/daemons/based/cibmon.c
	index 8fd249e8bb..c9b0784617 100644
	--- a/daemons/based/cibmon.c
	+++ b/daemons/based/cibmon.c
	@@ -1,261 +1,251 @@
	-
	-/*
	- * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	- *
	- * This program is free software; you can redistribute it and/or
	- * modify it under the terms of the GNU General Public
	- * License as published by the Free Software Foundation; either
	- * version 2 of the License, or (at your option) any later version.
	- *
	- * This software is distributed in the hope that it will be useful,
	- * but WITHOUT ANY WARRANTY; without even the implied warranty of
	- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	- * General Public License for more details.
	- *
	- * You should have received a copy of the GNU General Public
	- * License along with this library; if not, write to the Free Software
	- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	+/*
	+ * Copyright 2004-2019 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/cib/internal.h>

	#include <crm/common/ipc.h>
	#include <crm/pengine/status.h>
	#include <../lib/pengine/unpack.h>

	#include <crm/cib.h>

	#ifdef HAVE_GETOPT_H
	# include <getopt.h>
	#endif

	static int max_failures = 30;

	static gboolean log_diffs = FALSE;
	static gboolean log_updates = FALSE;

	static GMainLoop *mainloop = NULL;
	void usage(const char *cmd, crm_exit_t exit_status);
	void cib_connection_destroy(gpointer user_data);

	void cibmon_shutdown(int nsig);
	void cibmon_diff(const char event, xmlNode msg);

	static cib_t *cib = NULL;
	static xmlNode *cib_copy = NULL;

	#define OPTARGS "V?m:du"

	int
	main(int argc, char **argv)
	{
	int argerr = 0;
	int flag;
	int attempts = 0;
	int rc = pcmk_ok;

	#ifdef HAVE_GETOPT_H
	int option_index = 0;

	static struct option long_options[] = {
	/* Top-level Options */
	{"verbose", 0, 0, 'V'},
	{"help", 0, 0, '?'},
	{"log-diffs", 0, 0, 'd'},
	{"log-updates", 0, 0, 'u'},
	{"max-conn-fail", 1, 0, 'm'},
	{0, 0, 0, 0}
	};
	#endif

	crm_log_cli_init("cibmon");

	- crm_signal(SIGTERM, cibmon_shutdown);
	+ crm_signal_handler(SIGTERM, cibmon_shutdown);

	while (1) {
	#ifdef HAVE_GETOPT_H
	flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index);
	#else
	flag = getopt(argc, argv, OPTARGS);
	#endif
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case '?':
	usage(crm_system_name, CRM_EX_OK);
	break;
	case 'd':
	log_diffs = TRUE;
	break;
	case 'u':
	log_updates = TRUE;
	break;
	case 'm':
	max_failures = crm_parse_int(optarg, "30");
	break;
	default:
	printf("Argument code 0%o (%c)" " is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	usage(crm_system_name, CRM_EX_USAGE);
	}

	cib = cib_new();

	do {
	sleep(1);
	rc = cib->cmds->signon(cib, crm_system_name, cib_query);

	} while (rc == -ENOTCONN && attempts++ < max_failures);

	if (rc != pcmk_ok) {
	crm_err("Signon to CIB failed: %s", pcmk_strerror(rc));
	goto fail;
	}

	crm_debug("Setting dnotify");
	rc = cib->cmds->set_connection_dnotify(cib, cib_connection_destroy);
	if (rc != pcmk_ok) {
	crm_err("Failed to set dnotify callback: %s", pcmk_strerror(rc));
	goto fail;
	}

	crm_debug("Setting diff callback");
	rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, cibmon_diff);
	if (rc != pcmk_ok) {
	crm_err("Failed to set diff callback: %s", pcmk_strerror(rc));
	goto fail;
	}

	mainloop = g_main_loop_new(NULL, FALSE);
	crm_info("Starting mainloop");
	g_main_loop_run(mainloop);
	crm_trace("%s exiting normally", crm_system_name);
	fflush(stderr);
	return CRM_EX_OK;

	fail:
	crm_err("Setup failed, could not monitor CIB actions");
	return CRM_EX_ERROR;
	}

	void
	usage(const char *cmd, crm_exit_t exit_status)
	{
	FILE *stream;

	stream = (exit_status == CRM_EX_OK)? stdout : stderr;
	fflush(stream);

	crm_exit(exit_status);
	}

	void
	cib_connection_destroy(gpointer user_data)
	{
	cib_t *conn = user_data;

	crm_err("Connection to the CIB terminated... exiting");
	conn->cmds->signoff(conn); /* Ensure IPC is cleaned up */
	g_main_loop_quit(mainloop);
	return;
	}

	void
	cibmon_diff(const char event, xmlNode msg)
	{
	int rc = -1;
	const char *op = NULL;
	unsigned int log_level = LOG_INFO;

	xmlNode *diff = NULL;
	xmlNode *cib_last = NULL;
	xmlNode *update = get_message_xml(msg, F_CIB_UPDATE);

	if (msg == NULL) {
	crm_err("NULL update");
	return;
	}

	crm_element_value_int(msg, F_CIB_RC, &rc);
	op = crm_element_value(msg, F_CIB_OPERATION);
	diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok) {
	log_level = LOG_WARNING;
	do_crm_log(log_level, "[%s] %s ABORTED: %s", event, op, pcmk_strerror(rc));
	return;
	}

	if (log_diffs) {
	xml_log_patchset(log_level, op, diff);
	}

	if (log_updates && update != NULL) {
	crm_log_xml_trace(update, "raw_update");
	}

	if (cib_copy != NULL) {
	cib_last = cib_copy;
	cib_copy = NULL;
	rc = cib_process_diff(op, cib_force_diff, NULL, NULL, diff, cib_last, &cib_copy, NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply, requesting full copy: %s", pcmk_strerror(rc));
	free_xml(cib_copy);
	cib_copy = NULL;
	}
	}

	if (cib_copy == NULL) {
	rc = cib->cmds->query(cib, NULL, &cib_copy, cib_scope_local \| cib_sync_call);
	}

	if(rc == -EACCES) {
	crm_exit(CRM_EX_INSUFFICIENT_PRIV);
	}

	free_xml(cib_last);
	}

	void
	cibmon_shutdown(int nsig)
	{
	crm_exit(CRM_EX_OK);
	}
	diff --git a/include/crm/common/mainloop.h b/include/crm/common/mainloop.h
	index 5d423cb0d2..85da1cd661 100644
	--- a/include/crm/common/mainloop.h
	+++ b/include/crm/common/mainloop.h
	@@ -1,134 +1,135 @@
	/*
	* Copyright 2009-2019 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_COMMON_MAINLOOP__H
	# define CRM_COMMON_MAINLOOP__H

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Wrappers for and extensions to glib mainloop
	* \ingroup core
	*/

	# include <signal.h> // sighandler_t
	# include <glib.h>

	enum mainloop_child_flags {
	/* don't kill pid group on timeout, only kill the pid */
	mainloop_leave_pid_group = 0x01,
	};

	typedef struct trigger_s crm_trigger_t;
	typedef struct mainloop_io_s mainloop_io_t;
	typedef struct mainloop_child_s mainloop_child_t;
	typedef struct mainloop_timer_s mainloop_timer_t;

	void mainloop_cleanup(void);

	crm_trigger_t mainloop_add_trigger(int priority, int (dispatch) (gpointer user_data),
	gpointer userdata);

	void mainloop_set_trigger(crm_trigger_t * source);

	void mainloop_trigger_complete(crm_trigger_t * trig);

	gboolean mainloop_destroy_trigger(crm_trigger_t * source);

	# ifndef HAVE_SIGHANDLER_T
	typedef void (*sighandler_t)(int);
	# endif

	-sighandler_t crm_signal(int sig, sighandler_t dispatch);
	+sighandler_t crm_signal_handler(int sig, sighandler_t dispatch);
	+gboolean crm_signal(int sig, void (*dispatch) (int sig)); // deprecated

	gboolean mainloop_add_signal(int sig, void (*dispatch) (int sig));

	gboolean mainloop_destroy_signal(int sig);

	bool mainloop_timer_running(mainloop_timer_t *t);

	void mainloop_timer_start(mainloop_timer_t *t);

	void mainloop_timer_stop(mainloop_timer_t *t);

	guint mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms);

	mainloop_timer_t mainloop_timer_add(const char name, guint period_ms, bool repeat, GSourceFunc cb, void *userdata);

	void mainloop_timer_del(mainloop_timer_t *t);


	# include <crm/common/ipc.h>
	# include <qb/qbipcs.h>

	struct ipc_client_callbacks {
	int (dispatch) (const char buffer, ssize_t length, gpointer userdata);
	void (*destroy) (gpointer);
	};

	qb_ipcs_service_t mainloop_add_ipc_server(const char name, enum qb_ipc_type type,
	struct qb_ipcs_service_handlers *callbacks);

	void mainloop_del_ipc_server(qb_ipcs_service_t * server);

	mainloop_io_t mainloop_add_ipc_client(const char name, int priority, size_t max_size,
	void userdata, struct ipc_client_callbacks callbacks);

	void mainloop_del_ipc_client(mainloop_io_t * client);

	crm_ipc_t mainloop_get_ipc_client(mainloop_io_t client);

	struct mainloop_fd_callbacks {
	int (*dispatch) (gpointer userdata);
	void (*destroy) (gpointer userdata);
	};

	mainloop_io_t mainloop_add_fd(const char name, int priority, int fd, void *userdata,
	struct mainloop_fd_callbacks *callbacks);

	void mainloop_del_fd(mainloop_io_t * client);

	/*
	* Create a new tracked process
	* To track a process group, use -pid
	*/
	void mainloop_child_add(pid_t pid,
	int timeout,
	const char *desc,
	void *userdata,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode));

	void mainloop_child_add_with_flags(pid_t pid,
	int timeout,
	const char *desc,
	void *userdata,
	enum mainloop_child_flags,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode));

	void mainloop_child_userdata(mainloop_child_t child);
	int mainloop_child_timeout(mainloop_child_t * child);
	const char mainloop_child_name(mainloop_child_t child);

	pid_t mainloop_child_pid(mainloop_child_t * child);
	void mainloop_clear_child_userdata(mainloop_child_t * child);
	gboolean mainloop_child_kill(pid_t pid);

	void pcmk_drain_main_loop(GMainLoop *mloop, guint timer_ms,
	bool (*check)(guint));

	# define G_PRIORITY_MEDIUM (G_PRIORITY_HIGH/2)

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/lib/common/logging.c b/lib/common/logging.c
	index 56b00467e4..60745977ba 100644
	--- a/lib/common/logging.c
	+++ b/lib/common/logging.c
	@@ -1,1035 +1,1035 @@
	/*
	* Copyright 2004-2019 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <ctype.h>
	#include <pwd.h>
	#include <grp.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>
	#include <bzlib.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/common/mainloop.h>

	unsigned int crm_log_priority = LOG_NOTICE;
	unsigned int crm_log_level = LOG_INFO;
	static gboolean crm_tracing_enabled(void);
	unsigned int crm_trace_nonlog = 0;
	bool crm_is_daemon = 0;

	GLogFunc glib_log_default;

	static void
	crm_glib_handler(const gchar * log_domain, GLogLevelFlags flags, const gchar * message,
	gpointer user_data)
	{
	int log_level = LOG_WARNING;
	GLogLevelFlags msg_level = (flags & G_LOG_LEVEL_MASK);
	static struct qb_log_callsite *glib_cs = NULL;

	if (glib_cs == NULL) {
	glib_cs = qb_log_callsite_get(__FUNCTION__, __FILE__, "glib-handler", LOG_DEBUG, __LINE__, crm_trace_nonlog);
	}


	switch (msg_level) {
	case G_LOG_LEVEL_CRITICAL:
	log_level = LOG_CRIT;

	if (crm_is_callsite_active(glib_cs, LOG_DEBUG, 0) == FALSE) {
	/* log and record how we got here */
	crm_abort(__FILE__, __FUNCTION__, __LINE__, message, TRUE, TRUE);
	}
	break;

	case G_LOG_LEVEL_ERROR:
	log_level = LOG_ERR;
	break;
	case G_LOG_LEVEL_MESSAGE:
	log_level = LOG_NOTICE;
	break;
	case G_LOG_LEVEL_INFO:
	log_level = LOG_INFO;
	break;
	case G_LOG_LEVEL_DEBUG:
	log_level = LOG_DEBUG;
	break;

	case G_LOG_LEVEL_WARNING:
	case G_LOG_FLAG_RECURSION:
	case G_LOG_FLAG_FATAL:
	case G_LOG_LEVEL_MASK:
	log_level = LOG_WARNING;
	break;
	}

	do_crm_log(log_level, "%s: %s", log_domain, message);
	}

	#ifndef NAME_MAX
	# define NAME_MAX 256
	#endif

	/*!
	* \internal
	* \brief Write out a blackbox (enabling blackboxes if needed)
	*
	* \param[in] nsig Signal number that was received
	*
	* \note This is a true signal handler, and so must be async-safe.
	*/
	static void
	crm_trigger_blackbox(int nsig)
	{
	if(nsig == SIGTRAP) {
	/* Turn it on if it wasn't already */
	crm_enable_blackbox(nsig);
	}
	crm_write_blackbox(nsig, NULL);
	}

	const char *
	daemon_option(const char *option)
	{
	char env_name[NAME_MAX];
	const char *value = NULL;

	snprintf(env_name, NAME_MAX, "PCMK_%s", option);
	value = getenv(env_name);
	if (value != NULL) {
	crm_trace("Found %s = %s", env_name, value);
	return value;
	}

	snprintf(env_name, NAME_MAX, "HA_%s", option);
	value = getenv(env_name);
	if (value != NULL) {
	crm_trace("Found %s = %s", env_name, value);
	return value;
	}

	crm_trace("Nothing found for %s", option);
	return NULL;
	}

	void
	set_daemon_option(const char option, const char value)
	{
	char env_name[NAME_MAX];

	snprintf(env_name, NAME_MAX, "PCMK_%s", option);
	if (value) {
	crm_trace("Setting %s to %s", env_name, value);
	setenv(env_name, value, 1);
	} else {
	crm_trace("Unsetting %s", env_name);
	unsetenv(env_name);
	}

	snprintf(env_name, NAME_MAX, "HA_%s", option);
	if (value) {
	crm_trace("Setting %s to %s", env_name, value);
	setenv(env_name, value, 1);
	} else {
	crm_trace("Unsetting %s", env_name);
	unsetenv(env_name);
	}
	}

	gboolean
	daemon_option_enabled(const char daemon, const char option)
	{
	const char *value = daemon_option(option);

	if (value != NULL && crm_is_true(value)) {
	return TRUE;

	} else if (value != NULL && strstr(value, daemon)) {
	return TRUE;
	}

	return FALSE;
	}

	void
	crm_log_deinit(void)
	{
	g_log_set_default_handler(glib_log_default, NULL);
	}

	#define FMT_MAX 256

	static void
	set_format_string(int method, const char *daemon)
	{
	if (method == QB_LOG_SYSLOG) {
	// The system log gets a simplified, user-friendly format
	crm_extended_logging(method, QB_FALSE);
	qb_log_format_set(method, "%g %p: %b");

	} else {
	// Everything else gets more detail, for advanced troubleshooting

	int offset = 0;
	char fmt[FMT_MAX];

	if (method > QB_LOG_STDERR) {
	struct utsname res;
	const char *nodename = "localhost";

	if (uname(&res) == 0) {
	nodename = res.nodename;
	}

	// If logging to file, prefix with timestamp, node name, daemon ID
	offset += snprintf(fmt + offset, FMT_MAX - offset,
	"%%t %s %-20s[%lu] ",
	nodename, daemon, (unsigned long) getpid());
	}

	// Add function name (in parentheses)
	offset += snprintf(fmt + offset, FMT_MAX - offset, "(%%n");
	if (crm_tracing_enabled()) {
	// When tracing, add file and line number
	offset += snprintf(fmt + offset, FMT_MAX - offset, "@%%f:%%l");
	}
	offset += snprintf(fmt + offset, FMT_MAX - offset, ")");

	// Add tag (if any), severity, and actual message
	offset += snprintf(fmt + offset, FMT_MAX - offset, " %%g\t%%p: %%b");

	CRM_LOG_ASSERT(offset > 0);
	qb_log_format_set(method, fmt);
	}
	}

	gboolean
	crm_add_logfile(const char *filename)
	{
	bool is_default = false;
	static int default_fd = -1;
	static gboolean have_logfile = FALSE;
	const char *default_logfile = CRM_LOG_DIR "/pacemaker.log";

	struct stat parent;
	int fd = 0, rc = 0;
	FILE *logfile = NULL;
	char *parent_dir = NULL;
	char *filename_cp;

	if (filename == NULL && have_logfile == FALSE) {
	filename = default_logfile;
	}

	if (filename == NULL) {
	return FALSE; /* Nothing to do */
	} else if(safe_str_eq(filename, "none")) {
	return FALSE; /* Nothing to do */
	} else if(safe_str_eq(filename, "/dev/null")) {
	return FALSE; /* Nothing to do */
	} else if(safe_str_eq(filename, default_logfile)) {
	is_default = TRUE;
	}

	if(is_default && default_fd >= 0) {
	return TRUE; /* Nothing to do */
	}

	/* Check the parent directory */
	filename_cp = strdup(filename);
	parent_dir = dirname(filename_cp);
	rc = stat(parent_dir, &parent);

	if (rc != 0) {
	crm_err("Directory '%s' does not exist: logging to '%s' is disabled", parent_dir, filename);
	free(filename_cp);
	return FALSE;
	}
	free(filename_cp);

	errno = 0;
	logfile = fopen(filename, "a");
	if(logfile == NULL) {
	crm_err("%s (%d): Logging to '%s' as uid=%u, gid=%u is disabled",
	pcmk_strerror(errno), errno, filename, geteuid(), getegid());
	return FALSE;
	}

	/* Check/Set permissions if we're root */
	if (geteuid() == 0) {
	struct stat st;
	uid_t pcmk_uid = 0;
	gid_t pcmk_gid = 0;
	gboolean fix = FALSE;
	int logfd = fileno(logfile);

	rc = fstat(logfd, &st);
	if (rc < 0) {
	crm_perror(LOG_WARNING, "Cannot stat %s", filename);
	fclose(logfile);
	return FALSE;
	}

	if(crm_user_lookup(CRM_DAEMON_USER, &pcmk_uid, &pcmk_gid) == 0) {
	if (st.st_gid != pcmk_gid) {
	/* Wrong group */
	fix = TRUE;
	} else if ((st.st_mode & S_IRWXG) != (S_IRGRP \| S_IWGRP)) {
	/* Not read/writable by the correct group */
	fix = TRUE;
	}
	}

	if (fix) {
	rc = fchown(logfd, pcmk_uid, pcmk_gid);
	if (rc < 0) {
	crm_warn("Cannot change the ownership of %s to user %s and gid %d",
	filename, CRM_DAEMON_USER, pcmk_gid);
	}

	rc = fchmod(logfd, S_IRUSR \| S_IWUSR \| S_IRGRP \| S_IWGRP);
	if (rc < 0) {
	crm_warn("Cannot change the mode of %s to rw-rw----", filename);
	}

	fprintf(logfile, "Set r/w permissions for uid=%d, gid=%d on %s\n",
	pcmk_uid, pcmk_gid, filename);
	if (fflush(logfile) < 0 \|\| fsync(logfd) < 0) {
	crm_err("Couldn't write out logfile: %s", filename);
	}
	}
	}

	/* Close and reopen with libqb */
	fclose(logfile);
	fd = qb_log_file_open(filename);

	if (fd < 0) {
	crm_perror(LOG_WARNING, "Couldn't send additional logging to %s", filename);
	return FALSE;
	}

	if(is_default) {
	default_fd = fd;

	} else if(default_fd >= 0) {
	crm_notice("Switching to %s", filename);
	qb_log_ctl(default_fd, QB_LOG_CONF_ENABLED, QB_FALSE);
	}

	crm_notice("Additional logging available in %s", filename);
	qb_log_ctl(fd, QB_LOG_CONF_ENABLED, QB_TRUE);
	/* qb_log_ctl(fd, QB_LOG_CONF_FILE_SYNC, 1); Turn on synchronous writes */

	#ifdef HAVE_qb_log_conf_QB_LOG_CONF_MAX_LINE_LEN
	// Longer than default, for logging long XML lines
	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_MAX_LINE_LEN, 800);
	#endif

	/* Enable callsites */
	crm_update_callsites();
	have_logfile = TRUE;

	return TRUE;
	}

	static int blackbox_trigger = 0;
	static volatile char *blackbox_file_prefix = NULL;

	#ifdef QB_FEATURE_LOG_HIRES_TIMESTAMPS
	typedef struct timespec *log_time_t;
	#else
	typedef time_t log_time_t;
	#endif

	static void
	blackbox_logger(int32_t t, struct qb_log_callsite *cs, log_time_t timestamp,
	const char *msg)
	{
	if(cs && cs->priority < LOG_ERR) {
	crm_write_blackbox(SIGTRAP, cs); /* Bypass the over-dumping logic */
	} else {
	crm_write_blackbox(0, cs);
	}
	}

	static void
	crm_control_blackbox(int nsig, bool enable)
	{
	int lpc = 0;

	if (blackbox_file_prefix == NULL) {
	pid_t pid = getpid();

	blackbox_file_prefix = crm_strdup_printf("%s/%s-%lu",
	CRM_BLACKBOX_DIR,
	crm_system_name,
	(unsigned long) pid);
	}

	if (enable && qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_SIZE, 5 * 1024 * 1024); /* Any size change drops existing entries */
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE); /* Setting the size seems to disable it */

	/* Enable synchronous logging */
	for (lpc = QB_LOG_BLACKBOX; lpc < QB_LOG_TARGET_MAX; lpc++) {
	qb_log_ctl(lpc, QB_LOG_CONF_FILE_SYNC, QB_TRUE);
	}

	crm_notice("Initiated blackbox recorder: %s", blackbox_file_prefix);

	/* Save to disk on abnormal termination */
	- crm_signal(SIGSEGV, crm_trigger_blackbox);
	- crm_signal(SIGABRT, crm_trigger_blackbox);
	- crm_signal(SIGILL, crm_trigger_blackbox);
	- crm_signal(SIGBUS, crm_trigger_blackbox);
	- crm_signal(SIGFPE, crm_trigger_blackbox);
	+ crm_signal_handler(SIGSEGV, crm_trigger_blackbox);
	+ crm_signal_handler(SIGABRT, crm_trigger_blackbox);
	+ crm_signal_handler(SIGILL, crm_trigger_blackbox);
	+ crm_signal_handler(SIGBUS, crm_trigger_blackbox);
	+ crm_signal_handler(SIGFPE, crm_trigger_blackbox);

	crm_update_callsites();

	blackbox_trigger = qb_log_custom_open(blackbox_logger, NULL, NULL, NULL);
	qb_log_ctl(blackbox_trigger, QB_LOG_CONF_ENABLED, QB_TRUE);
	crm_trace("Trigger: %d is %d %d", blackbox_trigger,
	qb_log_ctl(blackbox_trigger, QB_LOG_CONF_STATE_GET, 0), QB_LOG_STATE_ENABLED);

	crm_update_callsites();

	} else if (!enable && qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0) == QB_LOG_STATE_ENABLED) {
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);

	/* Disable synchronous logging again when the blackbox is disabled */
	for (lpc = QB_LOG_BLACKBOX; lpc < QB_LOG_TARGET_MAX; lpc++) {
	qb_log_ctl(lpc, QB_LOG_CONF_FILE_SYNC, QB_FALSE);
	}
	}
	}

	void
	crm_enable_blackbox(int nsig)
	{
	crm_control_blackbox(nsig, TRUE);
	}

	void
	crm_disable_blackbox(int nsig)
	{
	crm_control_blackbox(nsig, FALSE);
	}

	/*!
	* \internal
	* \brief Write out a blackbox, if blackboxes are enabled
	*
	* \param[in] nsig Signal that was received
	* \param[in] cs libqb callsite
	*
	* \note This may be called via a true signal handler and so must be async-safe.
	* @TODO actually make this async-safe
	*/
	void
	crm_write_blackbox(int nsig, struct qb_log_callsite *cs)
	{
	static volatile int counter = 1;
	static volatile time_t last = 0;

	char buffer[NAME_MAX];
	time_t now = time(NULL);

	if (blackbox_file_prefix == NULL) {
	return;
	}

	switch (nsig) {
	case 0:
	case SIGTRAP:
	/* The graceful case - such as assertion failure or user request */

	if (nsig == 0 && now == last) {
	/* Prevent over-dumping */
	return;
	}

	snprintf(buffer, NAME_MAX, "%s.%d", blackbox_file_prefix, counter++);
	if (nsig == SIGTRAP) {
	crm_notice("Blackbox dump requested, please see %s for contents", buffer);

	} else if (cs) {
	syslog(LOG_NOTICE,
	"Problem detected at %s:%d (%s), please see %s for additional details",
	cs->function, cs->lineno, cs->filename, buffer);
	} else {
	crm_notice("Problem detected, please see %s for additional details", buffer);
	}

	last = now;
	qb_log_blackbox_write_to_file(buffer);

	/* Flush the existing contents
	* A size change would also work
	*/
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
	break;

	default:
	/* Do as little as possible, just try to get what we have out
	* We logged the filename when the blackbox was enabled
	*/
	- crm_signal(nsig, SIG_DFL);
	+ crm_signal_handler(nsig, SIG_DFL);
	qb_log_blackbox_write_to_file((const char *)blackbox_file_prefix);
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
	raise(nsig);
	break;
	}
	}

	gboolean
	crm_log_cli_init(const char *entity)
	{
	return crm_log_init(entity, LOG_ERR, FALSE, FALSE, 0, NULL, TRUE);
	}

	static const char *
	crm_quark_to_string(uint32_t tag)
	{
	const char *text = g_quark_to_string(tag);

	if (text) {
	return text;
	}
	return "";
	}

	static void
	crm_log_filter_source(int source, const char trace_files, const char trace_fns,
	const char trace_fmts, const char trace_tags, const char *trace_blackbox,
	struct qb_log_callsite *cs)
	{
	if (qb_log_ctl(source, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
	return;
	} else if (cs->tags != crm_trace_nonlog && source == QB_LOG_BLACKBOX) {
	/* Blackbox gets everything if enabled */
	qb_bit_set(cs->targets, source);

	} else if (source == blackbox_trigger && blackbox_trigger > 0) {
	/* Should this log message result in the blackbox being dumped */
	if (cs->priority <= LOG_ERR) {
	qb_bit_set(cs->targets, source);

	} else if (trace_blackbox) {
	char *key = crm_strdup_printf("%s:%d", cs->function, cs->lineno);

	if (strstr(trace_blackbox, key) != NULL) {
	qb_bit_set(cs->targets, source);
	}
	free(key);
	}

	} else if (source == QB_LOG_SYSLOG) { /* No tracing to syslog */
	if (cs->priority <= crm_log_priority && cs->priority <= crm_log_level) {
	qb_bit_set(cs->targets, source);
	}
	/* Log file tracing options... */
	} else if (cs->priority <= crm_log_level) {
	qb_bit_set(cs->targets, source);
	} else if (trace_files && strstr(trace_files, cs->filename) != NULL) {
	qb_bit_set(cs->targets, source);
	} else if (trace_fns && strstr(trace_fns, cs->function) != NULL) {
	qb_bit_set(cs->targets, source);
	} else if (trace_fmts && strstr(trace_fmts, cs->format) != NULL) {
	qb_bit_set(cs->targets, source);
	} else if (trace_tags
	&& cs->tags != 0
	&& cs->tags != crm_trace_nonlog && g_quark_to_string(cs->tags) != NULL) {
	qb_bit_set(cs->targets, source);
	}
	}

	static void
	crm_log_filter(struct qb_log_callsite *cs)
	{
	int lpc = 0;
	static int need_init = 1;
	static const char *trace_fns = NULL;
	static const char *trace_tags = NULL;
	static const char *trace_fmts = NULL;
	static const char *trace_files = NULL;
	static const char *trace_blackbox = NULL;

	if (need_init) {
	need_init = 0;
	trace_fns = getenv("PCMK_trace_functions");
	trace_fmts = getenv("PCMK_trace_formats");
	trace_tags = getenv("PCMK_trace_tags");
	trace_files = getenv("PCMK_trace_files");
	trace_blackbox = getenv("PCMK_trace_blackbox");

	if (trace_tags != NULL) {
	uint32_t tag;
	char token[500];
	const char *offset = NULL;
	const char *next = trace_tags;

	do {
	offset = next;
	next = strchrnul(offset, ',');
	snprintf(token, sizeof(token), "%.*s", (int)(next - offset), offset);

	tag = g_quark_from_string(token);
	crm_info("Created GQuark %u from token '%s' in '%s'", tag, token, trace_tags);

	if (next[0] != 0) {
	next++;
	}

	} while (next != NULL && next[0] != 0);
	}
	}

	cs->targets = 0; /* Reset then find targets to enable */
	for (lpc = QB_LOG_SYSLOG; lpc < QB_LOG_TARGET_MAX; lpc++) {
	crm_log_filter_source(lpc, trace_files, trace_fns, trace_fmts, trace_tags, trace_blackbox,
	cs);
	}
	}

	gboolean
	crm_is_callsite_active(struct qb_log_callsite *cs, uint8_t level, uint32_t tags)
	{
	gboolean refilter = FALSE;

	if (cs == NULL) {
	return FALSE;
	}

	if (cs->priority != level) {
	cs->priority = level;
	refilter = TRUE;
	}

	if (cs->tags != tags) {
	cs->tags = tags;
	refilter = TRUE;
	}

	if (refilter) {
	crm_log_filter(cs);
	}

	if (cs->targets == 0) {
	return FALSE;
	}
	return TRUE;
	}

	void
	crm_update_callsites(void)
	{
	static gboolean log = TRUE;

	if (log) {
	log = FALSE;
	crm_debug
	("Enabling callsites based on priority=%d, files=%s, functions=%s, formats=%s, tags=%s",
	crm_log_level, getenv("PCMK_trace_files"), getenv("PCMK_trace_functions"),
	getenv("PCMK_trace_formats"), getenv("PCMK_trace_tags"));
	}
	qb_log_filter_fn_set(crm_log_filter);
	}

	static gboolean
	crm_tracing_enabled(void)
	{
	if (crm_log_level >= LOG_TRACE) {
	return TRUE;
	} else if (getenv("PCMK_trace_files") \|\| getenv("PCMK_trace_functions")
	\|\| getenv("PCMK_trace_formats") \|\| getenv("PCMK_trace_tags")) {
	return TRUE;
	}
	return FALSE;
	}

	static int
	crm_priority2int(const char *name)
	{
	struct syslog_names {
	const char *name;
	int priority;
	};
	static struct syslog_names p_names[] = {
	{"emerg", LOG_EMERG},
	{"alert", LOG_ALERT},
	{"crit", LOG_CRIT},
	{"error", LOG_ERR},
	{"warning", LOG_WARNING},
	{"notice", LOG_NOTICE},
	{"info", LOG_INFO},
	{"debug", LOG_DEBUG},
	{NULL, -1}
	};
	int lpc;

	for (lpc = 0; name != NULL && p_names[lpc].name != NULL; lpc++) {
	if (crm_str_eq(p_names[lpc].name, name, TRUE)) {
	return p_names[lpc].priority;
	}
	}
	return crm_log_priority;
	}


	static void
	crm_identity(const char entity, int argc, char *argv)
	{
	if(crm_system_name != NULL) {
	/* Nothing to do */

	} else if (entity) {
	free(crm_system_name);
	crm_system_name = strdup(entity);

	} else if (argc > 0 && argv != NULL) {
	char *mutable = strdup(argv[0]);
	char *modified = basename(mutable);

	if (strstr(modified, "lt-") == modified) {
	modified += 3;
	}

	free(crm_system_name);
	crm_system_name = strdup(modified);
	free(mutable);

	} else if (crm_system_name == NULL) {
	crm_system_name = strdup("Unknown");
	}

	setenv("PCMK_service", crm_system_name, 1);
	}


	void
	crm_log_preinit(const char entity, int argc, char *argv)
	{
	/* Configure libqb logging with nothing turned on */

	int lpc = 0;
	int32_t qb_facility = 0;

	static bool have_logging = FALSE;

	if(have_logging == FALSE) {
	have_logging = TRUE;

	crm_xml_init(); /* Sets buffer allocation strategy */

	if (crm_trace_nonlog == 0) {
	crm_trace_nonlog = g_quark_from_static_string("Pacemaker non-logging tracepoint");
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	/* Redirect messages from glib functions to our handler */
	glib_log_default = g_log_set_default_handler(crm_glib_handler, NULL);

	/* and for good measure... - this enum is a bit field (!) */
	g_log_set_always_fatal((GLogLevelFlags) 0); /value out of range /

	/* Who do we log as */
	crm_identity(entity, argc, argv);

	qb_facility = qb_log_facility2int("local0");
	qb_log_init(crm_system_name, qb_facility, LOG_ERR);
	crm_log_level = LOG_CRIT;

	/* Nuke any syslog activity until it's asked for */
	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE);
	#ifdef HAVE_qb_log_conf_QB_LOG_CONF_MAX_LINE_LEN
	// Shorter than default, generous for what we should send to syslog
	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_MAX_LINE_LEN, 256);
	#endif

	/* Set format strings and disable threading
	* Pacemaker and threads do not mix well (due to the amount of forking)
	*/
	qb_log_tags_stringify_fn_set(crm_quark_to_string);
	for (lpc = QB_LOG_SYSLOG; lpc < QB_LOG_TARGET_MAX; lpc++) {
	qb_log_ctl(lpc, QB_LOG_CONF_THREADED, QB_FALSE);
	#ifdef HAVE_qb_log_conf_QB_LOG_CONF_ELLIPSIS
	// End truncated lines with '...'
	qb_log_ctl(lpc, QB_LOG_CONF_ELLIPSIS, QB_TRUE);
	#endif
	set_format_string(lpc, crm_system_name);
	}
	}
	}

	gboolean
	crm_log_init(const char *entity, uint8_t level, gboolean daemon, gboolean to_stderr,
	int argc, char **argv, gboolean quiet)
	{
	const char *syslog_priority = NULL;
	const char *logfile = daemon_option("logfile");
	const char *facility = daemon_option("logfacility");
	const char *f_copy = facility;

	crm_is_daemon = daemon;
	crm_log_preinit(entity, argc, argv);

	if(level > crm_log_level) {
	crm_log_level = level;
	}

	/* Should we log to syslog */
	if (facility == NULL) {
	if(crm_is_daemon) {
	facility = "daemon";
	} else {
	facility = "none";
	}
	set_daemon_option("logfacility", facility);
	}

	if (safe_str_eq(facility, "none")) {
	quiet = TRUE;


	} else {
	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_FACILITY, qb_log_facility2int(facility));
	}

	if (daemon_option_enabled(crm_system_name, "debug")) {
	/* Override the default setting */
	crm_log_level = LOG_DEBUG;
	}

	/* What lower threshold do we have for sending to syslog */
	syslog_priority = daemon_option("logpriority");
	if(syslog_priority) {
	int priority = crm_priority2int(syslog_priority);
	crm_log_priority = priority;
	qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*", priority);
	} else {
	qb_log_filter_ctl(QB_LOG_SYSLOG, QB_LOG_FILTER_ADD, QB_LOG_FILTER_FILE, "*", LOG_NOTICE);
	}

	// Log to syslog unless requested to be quiet
	if (!quiet) {
	qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_TRUE);
	}

	/* Should we log to stderr */
	if (daemon_option_enabled(crm_system_name, "stderr")) {
	/* Override the default setting */
	to_stderr = TRUE;
	}
	crm_enable_stderr(to_stderr);

	/* Should we log to a file */
	if (safe_str_eq("none", logfile)) {
	/* No soup^Hlogs for you! */
	} else if(crm_is_daemon) {
	// Daemons always get a log file, unless explicitly set to "none"
	crm_add_logfile(logfile);
	} else if(logfile) {
	crm_add_logfile(logfile);
	}

	if (crm_is_daemon && daemon_option_enabled(crm_system_name, "blackbox")) {
	crm_enable_blackbox(0);
	}

	/* Summary */
	crm_trace("Quiet: %d, facility %s", quiet, f_copy);
	daemon_option("logfile");
	daemon_option("logfacility");

	crm_update_callsites();

	/* Ok, now we can start logging... */
	if (quiet == FALSE && crm_is_daemon == FALSE) {
	crm_log_args(argc, argv);
	}

	if (crm_is_daemon) {
	const char *user = getenv("USER");

	if (user != NULL && safe_str_neq(user, "root") && safe_str_neq(user, CRM_DAEMON_USER)) {
	crm_trace("Not switching to corefile directory for %s", user);
	crm_is_daemon = FALSE;
	}
	}

	if (crm_is_daemon) {
	int user = getuid();
	const char *base = CRM_CORE_DIR;
	struct passwd *pwent = getpwuid(user);

	if (pwent == NULL) {
	crm_perror(LOG_ERR, "Cannot get name for uid: %d", user);

	} else if (safe_str_neq(pwent->pw_name, "root")
	&& safe_str_neq(pwent->pw_name, CRM_DAEMON_USER)) {
	crm_trace("Don't change active directory for regular user: %s", pwent->pw_name);

	} else if (chdir(base) < 0) {
	crm_perror(LOG_INFO, "Cannot change active directory to %s", base);

	} else {
	crm_info("Changed active directory to %s", base);
	#if 0
	{
	char path[512];

	snprintf(path, 512, "%s-%lu", crm_system_name, (unsigned long) getpid());
	mkdir(path, 0750);
	chdir(path);
	crm_info("Changed active directory to %s/%s/%s", base, pwent->pw_name, path);
	}
	#endif
	}

	/* Original meanings from signal(7)
	*
	* Signal Value Action Comment
	* SIGTRAP 5 Core Trace/breakpoint trap
	* SIGUSR1 30,10,16 Term User-defined signal 1
	* SIGUSR2 31,12,17 Term User-defined signal 2
	*
	* Our usage is as similar as possible
	*/
	mainloop_add_signal(SIGUSR1, crm_enable_blackbox);
	mainloop_add_signal(SIGUSR2, crm_disable_blackbox);
	mainloop_add_signal(SIGTRAP, crm_trigger_blackbox);
	}

	return TRUE;
	}

	/* returns the old value */
	unsigned int
	set_crm_log_level(unsigned int level)
	{
	unsigned int old = crm_log_level;

	crm_log_level = level;
	crm_update_callsites();
	crm_trace("New log level: %d", level);
	return old;
	}

	void
	crm_enable_stderr(int enable)
	{
	if (enable && qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) {
	qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_TRUE);
	crm_update_callsites();

	} else if (enable == FALSE) {
	qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_ENABLED, QB_FALSE);
	}
	}

	void
	crm_bump_log_level(int argc, char **argv)
	{
	static int args = TRUE;
	int level = crm_log_level;

	if (args && argc > 1) {
	crm_log_args(argc, argv);
	}

	if (qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_STATE_GET, 0) == QB_LOG_STATE_ENABLED) {
	set_crm_log_level(level + 1);
	}

	/* Enable after potentially logging the argstring, not before */
	crm_enable_stderr(TRUE);
	}

	unsigned int
	get_crm_log_level(void)
	{
	return crm_log_level;
	}

	#define ARGS_FMT "Invoked: %s"
	void
	crm_log_args(int argc, char **argv)
	{
	int lpc = 0;
	int len = 0;
	int existing_len = 0;
	int line = __LINE__;
	static int logged = 0;

	char *arg_string = NULL;

	if (argc == 0 \|\| argv == NULL \|\| logged) {
	return;
	}

	logged = 1;

	for (; lpc < argc; lpc++) {
	if (argv[lpc] == NULL) {
	break;
	}

	len = 2 + strlen(argv[lpc]); /* +1 space, +1 EOS */
	arg_string = realloc_safe(arg_string, len + existing_len);
	existing_len += sprintf(arg_string + existing_len, "%s ", argv[lpc]);
	}

	qb_log_from_external_source(__func__, __FILE__, ARGS_FMT, LOG_NOTICE, line, 0, arg_string);

	free(arg_string);
	}

	void
	crm_log_output_fn(const char file, const char function, int line, int level, const char *prefix,
	const char *output)
	{
	const char *next = NULL;
	const char *offset = NULL;

	if (output == NULL) {
	level = LOG_TRACE;
	output = "-- empty --";
	}

	next = output;
	do {
	offset = next;
	next = strchrnul(offset, '\n');
	do_crm_log_alias(level, file, function, line, "%s [ %.*s ]", prefix,
	(int)(next - offset), offset);
	if (next[0] != 0) {
	next++;
	}

	} while (next != NULL && next[0] != 0);
	}
	diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
	index 8b66f18ece..18f7014af1 100644
	--- a/lib/common/mainloop.c
	+++ b/lib/common/mainloop.c
	@@ -1,1341 +1,1351 @@
	/*
	* Copyright 2004-2019 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <stdlib.h>
	#include <signal.h>
	#include <errno.h>

	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipcs.h>

	#include <qb/qbarray.h>

	struct mainloop_child_s {
	pid_t pid;
	char *desc;
	unsigned timerid;
	gboolean timeout;
	void *privatedata;

	enum mainloop_child_flags flags;

	/* Called when a process dies */
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode);
	};

	struct trigger_s {
	GSource source;
	gboolean running;
	gboolean trigger;
	void *user_data;
	guint id;

	};

	static gboolean
	crm_trigger_prepare(GSource * source, gint * timeout)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	/* cluster-glue's FD and IPC related sources make use of
	* g_source_add_poll() but do not set a timeout in their prepare
	* functions
	*
	* This means mainloop's poll() will block until an event for one
	* of these sources occurs - any /other/ type of source, such as
	* this one or g_idle_*, that doesn't use g_source_add_poll() is
	* S-O-L and won't be processed until there is something fd-based
	* happens.
	*
	* Luckily the timeout we can set here affects all sources and
	* puts an upper limit on how long poll() can take.
	*
	* So unconditionally set a small-ish timeout, not too small that
	* we're in constant motion, which will act as an upper bound on
	* how long the signal handling might be delayed for.
	*/
	timeout = 500; / Timeout in ms */

	return trig->trigger;
	}

	static gboolean
	crm_trigger_check(GSource * source)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	return trig->trigger;
	}

	static gboolean
	crm_trigger_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	int rc = TRUE;
	crm_trigger_t trig = (crm_trigger_t ) source;

	if (trig->running) {
	/* Wait until the existing job is complete before starting the next one */
	return TRUE;
	}
	trig->trigger = FALSE;

	if (callback) {
	rc = callback(trig->user_data);
	if (rc < 0) {
	crm_trace("Trigger handler %p not yet complete", trig);
	trig->running = TRUE;
	rc = TRUE;
	}
	}
	return rc;
	}

	static void
	crm_trigger_finalize(GSource * source)
	{
	crm_trace("Trigger %p destroyed", source);
	}

	#if 0
	struct _GSourceCopy
	{
	gpointer callback_data;
	GSourceCallbackFuncs *callback_funcs;

	const GSourceFuncs *source_funcs;
	guint ref_count;

	GMainContext *context;

	gint priority;
	guint flags;
	guint source_id;

	GSList *poll_fds;

	GSource *prev;
	GSource *next;

	char *name;

	void *priv;
	};

	static int
	g_source_refcount(GSource * source)
	{
	/* Duplicating the contents of private header files is a necessary evil */
	if (source) {
	struct _GSourceCopy evil = (struct _GSourceCopy)source;
	return evil->ref_count;
	}
	return 0;
	}
	#else
	static int g_source_refcount(GSource * source)
	{
	return 0;
	}
	#endif

	static GSourceFuncs crm_trigger_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_trigger_dispatch,
	crm_trigger_finalize,
	};

	static crm_trigger_t *
	mainloop_setup_trigger(GSource * source, int priority, int (*dispatch) (gpointer user_data),
	gpointer userdata)
	{
	crm_trigger_t *trigger = NULL;

	trigger = (crm_trigger_t *) source;

	trigger->id = 0;
	trigger->trigger = FALSE;
	trigger->user_data = userdata;

	if (dispatch) {
	g_source_set_callback(source, dispatch, trigger, NULL);
	}

	g_source_set_priority(source, priority);
	g_source_set_can_recurse(source, FALSE);

	crm_trace("Setup %p with ref-count=%u", source, g_source_refcount(source));
	trigger->id = g_source_attach(source, NULL);
	crm_trace("Attached %p with ref-count=%u", source, g_source_refcount(source));

	return trigger;
	}

	void
	mainloop_trigger_complete(crm_trigger_t * trig)
	{
	crm_trace("Trigger handler %p complete", trig);
	trig->running = FALSE;
	}

	/* If dispatch returns:
	* -1: Job running but not complete
	* 0: Remove the trigger from mainloop
	* 1: Leave the trigger in mainloop
	*/
	crm_trigger_t *
	mainloop_add_trigger(int priority, int (*dispatch) (gpointer user_data), gpointer userdata)
	{
	GSource *source = NULL;

	CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
	source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
	CRM_ASSERT(source != NULL);

	return mainloop_setup_trigger(source, priority, dispatch, userdata);
	}

	void
	mainloop_set_trigger(crm_trigger_t * source)
	{
	if(source) {
	source->trigger = TRUE;
	}
	}

	gboolean
	mainloop_destroy_trigger(crm_trigger_t * source)
	{
	GSource *gs = NULL;

	if(source == NULL) {
	return TRUE;
	}

	gs = (GSource *)source;

	if(g_source_refcount(gs) > 2) {
	crm_info("Trigger %p is still referenced %u times", gs, g_source_refcount(gs));
	}

	g_source_destroy(gs); /* Remove from mainloop, ref_count-- */
	g_source_unref(gs); /* The caller no longer carries a reference to source
	*
	* At this point the source should be free'd,
	* unless we're currently processing said
	* source, in which case mainloop holds an
	* additional reference and it will be free'd
	* once our processing completes
	*/
	return TRUE;
	}

	// Define a custom glib source for signal handling

	// Data structure for custom glib source
	typedef struct signal_s {
	crm_trigger_t trigger; // trigger that invoked source (must be first)
	void (*handler) (int sig); // signal handler
	int signal; // signal that was received
	} crm_signal_t;

	// Table to associate signal handlers with signal numbers
	static crm_signal_t *crm_signals[NSIG];

	/*!
	* \internal
	* \brief Dispatch an event from custom glib source for signals
	*
	* Given an signal event, clear the event trigger and call any registered
	* signal handler.
	*
	* \param[in] source glib source that triggered this dispatch
	* \param[in] callback (ignored)
	* \param[in] userdata (ignored)
	*/
	static gboolean
	crm_signal_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	crm_signal_t sig = (crm_signal_t ) source;

	if(sig->signal != SIGCHLD) {
	crm_notice("Caught '%s' signal "CRM_XS" %d (%s handler)",
	strsignal(sig->signal), sig->signal,
	(sig->handler? "invoking" : "no"));
	}

	sig->trigger.trigger = FALSE;
	if (sig->handler) {
	sig->handler(sig->signal);
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Handle a signal by setting a trigger for signal source
	*
	* \param[in] sig Signal number that was received
	*
	* \note This is the true signal handler for the mainloop signal source, and
	* must be async-safe.
	*/
	static void
	mainloop_signal_handler(int sig)
	{
	if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
	mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
	}
	}

	// Functions implementing our custom glib source for signal handling
	static GSourceFuncs crm_signal_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_signal_dispatch,
	crm_trigger_finalize,
	};

	/*!
	* \internal
	* \brief Set a true signal handler
	*
	* signal()-like interface to sigaction()
	*
	* \param[in] sig Signal number to register handler for
	* \param[in] dispatch Signal handler
	*
	* \return The previous value of the signal handler, or SIG_ERR on error
	* \note The dispatch function must be async-safe.
	*/
	sighandler_t
	-crm_signal(int sig, sighandler_t dispatch)
	+crm_signal_handler(int sig, sighandler_t dispatch)
	{
	sigset_t mask;
	struct sigaction sa;
	struct sigaction old;

	if (sigemptyset(&mask) < 0) {
	crm_err("Could not set handler for signal %d: %s",
	sig, pcmk_strerror(errno));
	return SIG_ERR;
	}

	memset(&sa, 0, sizeof(struct sigaction));
	sa.sa_handler = dispatch;
	sa.sa_flags = SA_RESTART;
	sa.sa_mask = mask;

	if (sigaction(sig, &sa, &old) < 0) {
	crm_err("Could not set handler for signal %d: %s",
	sig, pcmk_strerror(errno));
	return SIG_ERR;
	}
	return old.sa_handler;
	}

	+/*
	+ * \brief Use crm_signal_handler() instead
	+ * \deprecated
	+ */
	+gboolean
	+crm_signal(int sig, void (*dispatch) (int sig))
	+{
	+ return crm_signal_handler(sig, dispatch) != SIG_ERR;
	+}
	+
	static void
	mainloop_destroy_signal_entry(int sig)
	{
	crm_signal_t *tmp = crm_signals[sig];

	crm_signals[sig] = NULL;

	crm_trace("Destroying signal %d", sig);
	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	}

	/*!
	* \internal
	* \brief Add a signal handler to a mainloop
	*
	* \param[in] sig Signal number to handle
	* \param[in] dispatch Signal handler function
	*
	* \note The true signal handler merely sets a mainloop trigger to call this
	* dispatch function via the mainloop. Therefore, the dispatch function
	* does not need to be async-safe.
	*/
	gboolean
	mainloop_add_signal(int sig, void (*dispatch) (int sig))
	{
	GSource *source = NULL;
	int priority = G_PRIORITY_HIGH - 1;

	if (sig == SIGTERM) {
	/* TERM is higher priority than other signals,
	* signals are higher priority than other ipc.
	* Yes, minus: smaller is "higher"
	*/
	priority--;
	}

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signals[sig] != NULL && crm_signals[sig]->handler == dispatch) {
	crm_trace("Signal handler for %d is already installed", sig);
	return TRUE;

	} else if (crm_signals[sig] != NULL) {
	crm_err("Different signal handler for %d is already installed", sig);
	return FALSE;
	}

	CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
	source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));

	crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
	CRM_ASSERT(crm_signals[sig] != NULL);

	crm_signals[sig]->handler = dispatch;
	crm_signals[sig]->signal = sig;

	- if (crm_signal(sig, mainloop_signal_handler) == SIG_ERR) {
	+ if (crm_signal_handler(sig, mainloop_signal_handler) == SIG_ERR) {
	mainloop_destroy_signal_entry(sig);
	return FALSE;
	}
	#if 0
	/* If we want signals to interrupt mainloop's poll(), instead of waiting for
	* the timeout, then we should call siginterrupt() below
	*
	* For now, just enforce a low timeout
	*/
	if (siginterrupt(sig, 1) < 0) {
	crm_perror(LOG_INFO, "Could not enable system call interruptions for signal %d", sig);
	}
	#endif

	return TRUE;
	}

	gboolean
	mainloop_destroy_signal(int sig)
	{
	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	- } else if (crm_signal(sig, NULL) == SIG_ERR) {
	+ } else if (crm_signal_handler(sig, NULL) == SIG_ERR) {
	crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
	return FALSE;

	} else if (crm_signals[sig] == NULL) {
	return TRUE;
	}
	mainloop_destroy_signal_entry(sig);
	return TRUE;
	}

	static qb_array_t *gio_map = NULL;

	void
	mainloop_cleanup(void)
	{
	if (gio_map) {
	qb_array_free(gio_map);
	}

	for (int sig = 0; sig < NSIG; ++sig) {
	mainloop_destroy_signal_entry(sig);
	}
	}

	/*
	* libqb...
	*/
	struct gio_to_qb_poll {
	int32_t is_used;
	guint source;
	int32_t events;
	void *data;
	qb_ipcs_dispatch_fn_t fn;
	enum qb_loop_priority p;
	};

	static gboolean
	gio_read_socket(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;
	gint fd = g_io_channel_unix_get_fd(gio);

	crm_trace("%p.%d %d", data, fd, condition);

	/* if this assert get's hit, then there is a race condition between
	* when we destroy a fd and when mainloop actually gives it up */
	CRM_ASSERT(adaptor->is_used > 0);

	return (adaptor->fn(fd, condition, adaptor->data) == 0);
	}

	static void
	gio_poll_destroy(gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;

	adaptor->is_used--;
	CRM_ASSERT(adaptor->is_used >= 0);

	if (adaptor->is_used == 0) {
	crm_trace("Marking adaptor %p unused", adaptor);
	adaptor->source = 0;
	}
	}

	static int32_t
	gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn, int32_t add)
	{
	struct gio_to_qb_poll *adaptor;
	GIOChannel *channel;
	int32_t res = 0;

	res = qb_array_index(gio_map, fd, (void **)&adaptor);
	if (res < 0) {
	crm_err("Array lookup failed for fd=%d: %d", fd, res);
	return res;
	}

	crm_trace("Adding fd=%d to mainloop as adaptor %p", fd, adaptor);

	if (add && adaptor->source) {
	crm_err("Adaptor for descriptor %d is still in-use", fd);
	return -EEXIST;
	}
	if (!add && !adaptor->is_used) {
	crm_err("Adaptor for descriptor %d is not in-use", fd);
	return -ENOENT;
	}

	/* channel is created with ref_count = 1 */
	channel = g_io_channel_unix_new(fd);
	if (!channel) {
	crm_err("No memory left to add fd=%d", fd);
	return -ENOMEM;
	}

	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}

	/* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
	evts \|= (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR);

	adaptor->fn = fn;
	adaptor->events = evts;
	adaptor->data = data;
	adaptor->p = p;
	adaptor->is_used++;
	adaptor->source =
	g_io_add_watch_full(channel, G_PRIORITY_DEFAULT, evts, gio_read_socket, adaptor,
	gio_poll_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after gio_poll_destroy() completes
	*/
	g_io_channel_unref(channel);

	crm_trace("Added to mainloop with gsource id=%d", adaptor->source);
	if (adaptor->source > 0) {
	return 0;
	}

	return -EINVAL;
	}

	static int32_t
	gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_TRUE);
	}

	static int32_t
	gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_FALSE);
	}

	static int32_t
	gio_poll_dispatch_del(int32_t fd)
	{
	struct gio_to_qb_poll *adaptor;

	crm_trace("Looking for fd=%d", fd);
	if (qb_array_index(gio_map, fd, (void **)&adaptor) == 0) {
	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}
	}
	return 0;
	}

	struct qb_ipcs_poll_handlers gio_poll_funcs = {
	.job_add = NULL,
	.dispatch_add = gio_poll_dispatch_add,
	.dispatch_mod = gio_poll_dispatch_mod,
	.dispatch_del = gio_poll_dispatch_del,
	};

	static enum qb_ipc_type
	pick_ipc_type(enum qb_ipc_type requested)
	{
	const char *env = getenv("PCMK_ipc_type");

	if (env && strcmp("shared-mem", env) == 0) {
	return QB_IPC_SHM;
	} else if (env && strcmp("socket", env) == 0) {
	return QB_IPC_SOCKET;
	} else if (env && strcmp("posix", env) == 0) {
	return QB_IPC_POSIX_MQ;
	} else if (env && strcmp("sysv", env) == 0) {
	return QB_IPC_SYSV_MQ;
	} else if (requested == QB_IPC_NATIVE) {
	/* We prefer shared memory because the server never blocks on
	* send. If part of a message fits into the socket, libqb
	* needs to block until the remainder can be sent also.
	* Otherwise the client will wait forever for the remaining
	* bytes.
	*/
	return QB_IPC_SHM;
	}
	return requested;
	}

	qb_ipcs_service_t *
	mainloop_add_ipc_server(const char *name, enum qb_ipc_type type,
	struct qb_ipcs_service_handlers * callbacks)
	{
	int rc = 0;
	qb_ipcs_service_t *server = NULL;

	if (gio_map == NULL) {
	gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
	}

	crm_client_init();
	server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);

	#ifdef HAVE_IPCS_GET_BUFFER_SIZE
	/* All clients should use at least ipc_buffer_max as their buffer size */
	qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size());
	#endif

	qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);

	rc = qb_ipcs_run(server);
	if (rc < 0) {
	crm_err("Could not start %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
	return NULL;
	}

	return server;
	}

	void
	mainloop_del_ipc_server(qb_ipcs_service_t * server)
	{
	if (server) {
	qb_ipcs_destroy(server);
	}
	}

	struct mainloop_io_s {
	char *name;
	void *userdata;

	int fd;
	guint source;
	crm_ipc_t *ipc;
	GIOChannel *channel;

	int (dispatch_fn_ipc) (const char buffer, ssize_t length, gpointer userdata);
	int (*dispatch_fn_io) (gpointer userdata);
	void (*destroy_fn) (gpointer userdata);

	};

	static gboolean
	mainloop_gio_callback(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	gboolean keep = TRUE;
	mainloop_io_t *client = data;

	CRM_ASSERT(client->fd == g_io_channel_unix_get_fd(gio));

	if (condition & G_IO_IN) {
	if (client->ipc) {
	long rc = 0;
	int max = 10;

	do {
	rc = crm_ipc_read(client->ipc);
	if (rc <= 0) {
	crm_trace("Message acquisition from %s[%p] failed: %s (%ld)",
	client->name, client, pcmk_strerror(rc), rc);

	} else if (client->dispatch_fn_ipc) {
	const char *buffer = crm_ipc_buffer(client->ipc);

	crm_trace("New message from %s[%p] = %ld (I/O condition=%d)", client->name, client, rc, condition);
	if (client->dispatch_fn_ipc(buffer, rc, client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}

	} while (keep && rc > 0 && --max > 0);

	} else {
	crm_trace("New message from %s[%p] %u", client->name, client, condition);
	if (client->dispatch_fn_io) {
	if (client->dispatch_fn_io(client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}
	}
	}

	if (client->ipc && crm_ipc_connected(client->ipc) == FALSE) {
	crm_err("Connection to %s closed " CRM_XS "client=%p condition=%d",
	client->name, client, condition);
	keep = FALSE;

	} else if (condition & (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR)) {
	crm_trace("The connection %s[%p] has been closed (I/O condition=%d)",
	client->name, client, condition);
	keep = FALSE;

	} else if ((condition & G_IO_IN) == 0) {
	/*
	#define GLIB_SYSDEF_POLLIN =1
	#define GLIB_SYSDEF_POLLPRI =2
	#define GLIB_SYSDEF_POLLOUT =4
	#define GLIB_SYSDEF_POLLERR =8
	#define GLIB_SYSDEF_POLLHUP =16
	#define GLIB_SYSDEF_POLLNVAL =32

	typedef enum
	{
	G_IO_IN GLIB_SYSDEF_POLLIN,
	G_IO_OUT GLIB_SYSDEF_POLLOUT,
	G_IO_PRI GLIB_SYSDEF_POLLPRI,
	G_IO_ERR GLIB_SYSDEF_POLLERR,
	G_IO_HUP GLIB_SYSDEF_POLLHUP,
	G_IO_NVAL GLIB_SYSDEF_POLLNVAL
	} GIOCondition;

	A bitwise combination representing a condition to watch for on an event source.

	G_IO_IN There is data to read.
	G_IO_OUT Data can be written (without blocking).
	G_IO_PRI There is urgent data to read.
	G_IO_ERR Error condition.
	G_IO_HUP Hung up (the connection has been broken, usually for pipes and sockets).
	G_IO_NVAL Invalid request. The file descriptor is not open.
	*/
	crm_err("Strange condition: %d", condition);
	}

	/* keep == FALSE results in mainloop_gio_destroy() being called
	* just before the source is removed from mainloop
	*/
	return keep;
	}

	static void
	mainloop_gio_destroy(gpointer c)
	{
	mainloop_io_t *client = c;
	char *c_name = strdup(client->name);

	/* client->source is valid but about to be destroyed (ref_count == 0) in gmain.c
	* client->channel will still have ref_count > 0... should be == 1
	*/
	crm_trace("Destroying client %s[%p]", c_name, c);

	if (client->ipc) {
	crm_ipc_close(client->ipc);
	}

	if (client->destroy_fn) {
	void (*destroy_fn) (gpointer userdata) = client->destroy_fn;

	client->destroy_fn = NULL;
	destroy_fn(client->userdata);
	}

	if (client->ipc) {
	crm_ipc_t *ipc = client->ipc;

	client->ipc = NULL;
	crm_ipc_destroy(ipc);
	}

	crm_trace("Destroyed client %s[%p]", c_name, c);

	free(client->name); client->name = NULL;
	free(client);

	free(c_name);
	}

	mainloop_io_t *
	mainloop_add_ipc_client(const char name, int priority, size_t max_size, void userdata,
	struct ipc_client_callbacks *callbacks)
	{
	mainloop_io_t *client = NULL;
	crm_ipc_t *conn = crm_ipc_new(name, max_size);

	if (conn && crm_ipc_connect(conn)) {
	int32_t fd = crm_ipc_get_fd(conn);

	client = mainloop_add_fd(name, priority, fd, userdata, NULL);
	}

	if (client == NULL) {
	crm_perror(LOG_TRACE, "Connection to %s failed", name);
	if (conn) {
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	}
	return NULL;
	}

	client->ipc = conn;
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_ipc = callbacks->dispatch;
	return client;
	}

	void
	mainloop_del_ipc_client(mainloop_io_t * client)
	{
	mainloop_del_fd(client);
	}

	crm_ipc_t *
	mainloop_get_ipc_client(mainloop_io_t * client)
	{
	if (client) {
	return client->ipc;
	}
	return NULL;
	}

	mainloop_io_t *
	mainloop_add_fd(const char name, int priority, int fd, void userdata,
	struct mainloop_fd_callbacks * callbacks)
	{
	mainloop_io_t *client = NULL;

	if (fd >= 0) {
	client = calloc(1, sizeof(mainloop_io_t));
	if (client == NULL) {
	return NULL;
	}
	client->name = strdup(name);
	client->userdata = userdata;

	if (callbacks) {
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_io = callbacks->dispatch;
	}

	client->fd = fd;
	client->channel = g_io_channel_unix_new(fd);
	client->source =
	g_io_add_watch_full(client->channel, priority,
	(G_IO_IN \| G_IO_HUP \| G_IO_NVAL \| G_IO_ERR), mainloop_gio_callback,
	client, mainloop_gio_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch() or g_source_remove()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after mainloop_gio_destroy() completes
	*/
	g_io_channel_unref(client->channel);
	crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
	} else {
	errno = EINVAL;
	}

	return client;
	}

	void
	mainloop_del_fd(mainloop_io_t * client)
	{
	if (client != NULL) {
	crm_trace("Removing client %s[%p]", client->name, client);
	if (client->source) {
	/* Results in mainloop_gio_destroy() being called just
	* before the source is removed from mainloop
	*/
	g_source_remove(client->source);
	}
	}
	}

	static GListPtr child_list = NULL;

	pid_t
	mainloop_child_pid(mainloop_child_t * child)
	{
	return child->pid;
	}

	const char *
	mainloop_child_name(mainloop_child_t * child)
	{
	return child->desc;
	}

	int
	mainloop_child_timeout(mainloop_child_t * child)
	{
	return child->timeout;
	}

	void *
	mainloop_child_userdata(mainloop_child_t * child)
	{
	return child->privatedata;
	}

	void
	mainloop_clear_child_userdata(mainloop_child_t * child)
	{
	child->privatedata = NULL;
	}

	/* good function name */
	static void
	child_free(mainloop_child_t *child)
	{
	if (child->timerid != 0) {
	crm_trace("Removing timer %d", child->timerid);
	g_source_remove(child->timerid);
	child->timerid = 0;
	}
	free(child->desc);
	free(child);
	}

	/* terrible function name */
	static int
	child_kill_helper(mainloop_child_t *child)
	{
	int rc;
	if (child->flags & mainloop_leave_pid_group) {
	crm_debug("Kill pid %d only. leave group intact.", child->pid);
	rc = kill(child->pid, SIGKILL);
	} else {
	crm_debug("Kill pid %d's group", child->pid);
	rc = kill(-child->pid, SIGKILL);
	}

	if (rc < 0) {
	if (errno != ESRCH) {
	crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
	}
	return -errno;
	}
	return 0;
	}

	static gboolean
	child_timeout_callback(gpointer p)
	{
	mainloop_child_t *child = p;
	int rc = 0;

	child->timerid = 0;
	if (child->timeout) {
	crm_crit("%s process (PID %d) will not die!", child->desc, (int)child->pid);
	return FALSE;
	}

	rc = child_kill_helper(child);
	if (rc == ESRCH) {
	/* Nothing left to do. pid doesn't exist */
	return FALSE;
	}

	child->timeout = TRUE;
	crm_warn("%s process (PID %d) timed out", child->desc, (int)child->pid);

	child->timerid = g_timeout_add(5000, child_timeout_callback, child);
	return FALSE;
	}

	static gboolean
	child_waitpid(mainloop_child_t *child, int flags)
	{
	int rc = 0;
	int core = 0;
	int signo = 0;
	int status = 0;
	int exitcode = 0;

	rc = waitpid(child->pid, &status, flags);
	if(rc == 0) {
	crm_perror(LOG_DEBUG, "wait(%d) = %d", child->pid, rc);
	return FALSE;

	} else if(rc != child->pid) {
	signo = SIGCHLD;
	exitcode = 1;
	status = 1;
	crm_perror(LOG_ERR, "Call to waitpid(%d) failed", child->pid);

	} else {
	crm_trace("Managed process %d exited: %p", child->pid, child);

	if (WIFEXITED(status)) {
	exitcode = WEXITSTATUS(status);
	crm_trace("Managed process %d (%s) exited with rc=%d", child->pid, child->desc, exitcode);

	} else if (WIFSIGNALED(status)) {
	signo = WTERMSIG(status);
	crm_trace("Managed process %d (%s) exited with signal=%d", child->pid, child->desc, signo);
	}
	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	core = 1;
	crm_err("Managed process %d (%s) dumped core", child->pid, child->desc);
	}
	#endif
	}

	if (child->callback) {
	child->callback(child, child->pid, core, signo, exitcode);
	}
	return TRUE;
	}

	static void
	child_death_dispatch(int signal)
	{
	GListPtr iter = child_list;
	gboolean exited;

	while(iter) {
	GListPtr saved = NULL;
	mainloop_child_t *child = iter->data;
	exited = child_waitpid(child, WNOHANG);

	saved = iter;
	iter = iter->next;

	if (exited == FALSE) {
	continue;
	}
	crm_trace("Removing process entry %p for %d", child, child->pid);

	child_list = g_list_remove_link(child_list, saved);
	g_list_free(saved);
	child_free(child);
	}
	}

	static gboolean
	child_signal_init(gpointer p)
	{
	crm_trace("Installed SIGCHLD handler");
	/* Do NOT use g_child_watch_add() and friends, they rely on pthreads */
	mainloop_add_signal(SIGCHLD, child_death_dispatch);

	/* In case they terminated before the signal handler was installed */
	child_death_dispatch(SIGCHLD);
	return FALSE;
	}

	int
	mainloop_child_kill(pid_t pid)
	{
	GListPtr iter;
	mainloop_child_t *child = NULL;
	mainloop_child_t *match = NULL;
	/* It is impossible to block SIGKILL, this allows us to
	* call waitpid without WNOHANG flag.*/
	int waitflags = 0, rc = 0;

	for (iter = child_list; iter != NULL && match == NULL; iter = iter->next) {
	child = iter->data;
	if (pid == child->pid) {
	match = child;
	}
	}

	if (match == NULL) {
	return FALSE;
	}

	rc = child_kill_helper(match);
	if(rc == -ESRCH) {
	/* It's gone, but hasn't shown up in waitpid() yet
	*
	* Wait until we get SIGCHLD and let child_death_dispatch()
	* clean it up as normal (so we get the correct return
	* code/status)
	*
	* The blocking alternative would be to call:
	* child_waitpid(match, 0);
	*/
	crm_trace("Waiting for child %d to be reaped by child_death_dispatch()", match->pid);
	return TRUE;

	} else if(rc != 0) {
	/* If KILL for some other reason set the WNOHANG flag since we
	* can't be certain what happened.
	*/
	waitflags = WNOHANG;
	}

	if (child_waitpid(match, waitflags) == FALSE) {
	/* not much we can do if this occurs */
	return FALSE;
	}

	child_list = g_list_remove(child_list, match);
	child_free(match);
	return TRUE;
	}

	/* Create/Log a new tracked process
	* To track a process group, use -pid
	*/
	void
	mainloop_child_add_with_flags(pid_t pid, int timeout, const char desc, void privatedata, enum mainloop_child_flags flags,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	static bool need_init = TRUE;
	mainloop_child_t *child = g_new(mainloop_child_t, 1);

	child->pid = pid;
	child->timerid = 0;
	child->timeout = FALSE;
	child->privatedata = privatedata;
	child->callback = callback;
	child->flags = flags;

	if(desc) {
	child->desc = strdup(desc);
	}

	if (timeout) {
	child->timerid = g_timeout_add(timeout, child_timeout_callback, child);
	}

	child_list = g_list_append(child_list, child);

	if(need_init) {
	need_init = FALSE;
	/* SIGCHLD processing has to be invoked from mainloop.
	* We do not want it to be possible to both add a child pid
	* to mainloop, and have the pid's exit callback invoked within
	* the same callstack. */
	g_timeout_add(1, child_signal_init, NULL);
	}
	}

	void
	mainloop_child_add(pid_t pid, int timeout, const char desc, void privatedata,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback);
	}

	struct mainloop_timer_s {
	guint id;
	guint period_ms;
	bool repeat;
	char *name;
	GSourceFunc cb;
	void *userdata;
	};

	static gboolean mainloop_timer_cb(gpointer user_data)
	{
	int id = 0;
	bool repeat = FALSE;
	struct mainloop_timer_s *t = user_data;

	CRM_ASSERT(t != NULL);

	id = t->id;
	t->id = 0; /* Ensure it's unset during callbacks so that
	* mainloop_timer_running() works as expected
	*/

	if(t->cb) {
	crm_trace("Invoking callbacks for timer %s", t->name);
	repeat = t->repeat;
	if(t->cb(t->userdata) == FALSE) {
	crm_trace("Timer %s complete", t->name);
	repeat = FALSE;
	}
	}

	if(repeat) {
	/* Restore if repeating */
	t->id = id;
	}

	return repeat;
	}

	bool mainloop_timer_running(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	return TRUE;
	}
	return FALSE;
	}

	void mainloop_timer_start(mainloop_timer_t *t)
	{
	mainloop_timer_stop(t);
	if(t && t->period_ms > 0) {
	crm_trace("Starting timer %s", t->name);
	t->id = g_timeout_add(t->period_ms, mainloop_timer_cb, t);
	}
	}

	void mainloop_timer_stop(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	crm_trace("Stopping timer %s", t->name);
	g_source_remove(t->id);
	t->id = 0;
	}
	}

	guint mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms)
	{
	guint last = 0;

	if(t) {
	last = t->period_ms;
	t->period_ms = period_ms;
	}

	if(t && t->id != 0 && last != t->period_ms) {
	mainloop_timer_start(t);
	}
	return last;
	}


	mainloop_timer_t *
	mainloop_timer_add(const char name, guint period_ms, bool repeat, GSourceFunc cb, void userdata)
	{
	mainloop_timer_t *t = calloc(1, sizeof(mainloop_timer_t));

	if(t) {
	if(name) {
	t->name = crm_strdup_printf("%s-%u-%d", name, period_ms, repeat);
	} else {
	t->name = crm_strdup_printf("%p-%u-%d", t, period_ms, repeat);
	}
	t->id = 0;
	t->period_ms = period_ms;
	t->repeat = repeat;
	t->cb = cb;
	t->userdata = userdata;
	crm_trace("Created timer %s with %p %p", t->name, userdata, t->userdata);
	}
	return t;
	}

	void
	mainloop_timer_del(mainloop_timer_t *t)
	{
	if(t) {
	crm_trace("Destroying timer %s", t->name);
	mainloop_timer_stop(t);
	free(t->name);
	free(t);
	}
	}

	/*
	* Helpers to make sure certain events aren't lost at shutdown
	*/

	static gboolean
	drain_timeout_cb(gpointer user_data)
	{
	bool timeout_popped = (bool) user_data;

	*timeout_popped = TRUE;
	return FALSE;
	}

	/*!
	* \brief Process main loop events while a certain condition is met
	*
	* \param[in] mloop Main loop to process
	* \param[in] timer_ms Don't process longer than this amount of time
	* \param[in] check Function that returns TRUE if events should be processed
	*
	* \note This function is intended to be called at shutdown if certain important
	* events should not be missed. The caller would likely quit the main loop
	* or exit after calling this function. The check() function will be
	* passed the remaining timeout in milliseconds.
	*/
	void
	pcmk_drain_main_loop(GMainLoop mloop, guint timer_ms, bool (check)(guint))
	{
	bool timeout_popped = FALSE;
	guint timer = 0;
	GMainContext *ctx = NULL;

	CRM_CHECK(mloop && check, return);

	ctx = g_main_loop_get_context(mloop);
	if (ctx) {
	time_t start_time = time(NULL);

	timer = g_timeout_add(timer_ms, drain_timeout_cb, &timeout_popped);
	while (!timeout_popped
	&& check(timer_ms - (time(NULL) - start_time) * 1000)) {
	g_main_context_iteration(ctx, TRUE);
	}
	}
	if (!timeout_popped && (timer > 0)) {
	g_source_remove(timer);
	}
	}
	diff --git a/tools/crm_mon.c b/tools/crm_mon.c
	index cb77ef5880..c6ce73b696 100644
	--- a/tools/crm_mon.c
	+++ b/tools/crm_mon.c
	@@ -1,4435 +1,4435 @@
	/*
	* Copyright 2004-2019 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <signal.h>
	#include <sys/utsname.h>

	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/lrmd.h>
	#include <crm/common/internal.h> /* crm_ends_with_ext */
	#include <crm/common/ipc.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/util.h>
	#include <crm/common/xml.h>

	#include <crm/cib/internal.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <../lib/pengine/unpack.h>
	#include <pacemaker-internal.h>
	#include <crm/stonith-ng.h>

	static void clean_up_connections(void);
	static crm_exit_t clean_up(crm_exit_t exit_code);
	static void crm_diff_update(const char event, xmlNode msg);
	static gboolean mon_refresh_display(gpointer user_data);
	static int cib_connect(gboolean full);
	static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
	static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
	static void kick_refresh(gboolean data_updated);
	static char get_node_display_name(node_t node);

	/*
	* Definitions indicating which items to print
	*/

	#define mon_show_times (0x0001U)
	#define mon_show_stack (0x0002U)
	#define mon_show_dc (0x0004U)
	#define mon_show_count (0x0008U)
	#define mon_show_nodes (0x0010U)
	#define mon_show_resources (0x0020U)
	#define mon_show_attributes (0x0040U)
	#define mon_show_failcounts (0x0080U)
	#define mon_show_operations (0x0100U)
	#define mon_show_tickets (0x0200U)
	#define mon_show_bans (0x0400U)
	#define mon_show_fence_history (0x0800U)

	#define mon_show_headers (mon_show_times \| mon_show_stack \| mon_show_dc \
	\| mon_show_count)
	#define mon_show_default (mon_show_headers \| mon_show_nodes \
	\| mon_show_resources)
	#define mon_show_all (mon_show_default \| mon_show_attributes \
	\| mon_show_failcounts \| mon_show_operations \
	\| mon_show_tickets \| mon_show_bans \
	\| mon_show_fence_history)

	static unsigned int show = mon_show_default;

	/*
	* Definitions indicating how to output
	*/

	enum mon_output_format_e {
	mon_output_none,
	mon_output_monitor,
	mon_output_plain,
	mon_output_console,
	mon_output_xml,
	mon_output_html,
	mon_output_cgi
	} output_format = mon_output_console;

	static char output_filename = NULL; / if sending output to a file, its name */

	/* other globals */
	static char *pid_file = NULL;

	static gboolean group_by_node = FALSE;
	static gboolean inactive_resources = FALSE;
	static int reconnect_msec = 5000;
	static gboolean daemonize = FALSE;
	static GMainLoop *mainloop = NULL;
	static guint timer_id = 0;
	static mainloop_timer_t *refresh_timer = NULL;
	static pe_working_set_t *mon_data_set = NULL;
	static GList *attr_list = NULL;

	static const char *external_agent = NULL;
	static const char *external_recipient = NULL;

	static cib_t *cib = NULL;
	static stonith_t *st = NULL;
	static xmlNode *current_cib = NULL;

	static gboolean one_shot = FALSE;
	static gboolean has_warnings = FALSE;
	static gboolean print_timing = FALSE;
	static gboolean watch_fencing = FALSE;
	static gboolean fence_history = FALSE;
	static gboolean fence_full_history = FALSE;
	static gboolean fence_connect = FALSE;
	static int fence_history_level = 1;
	static gboolean print_brief = FALSE;
	static gboolean print_pending = TRUE;
	static gboolean print_clone_detail = FALSE;
	#if CURSES_ENABLED
	static gboolean curses_console_initialized = FALSE;
	#endif

	/* FIXME allow, detect, and correctly interpret glob pattern or regex? */
	const char *print_neg_location_prefix = "";

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { CRM_FAIL_COUNT_PREFIX, CRM_LAST_FAILURE_PREFIX, \
	"shutdown", "terminate", "standby", "probe_complete", \
	"#", NULL }

	long last_refresh = 0;
	crm_trigger_t *refresh_trigger = NULL;

	/* Define exit codes for monitoring-compatible output
	* For nagios plugins, the possibilities are
	* OK=0, WARN=1, CRIT=2, and UNKNOWN=3
	*/
	#define MON_STATUS_WARN CRM_EX_ERROR
	#define MON_STATUS_CRIT CRM_EX_INVALID_PARAM
	#define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE

	/* Convenience macro for prettifying output (e.g. "node" vs "nodes") */
	#define s_if_plural(i) (((i) == 1)? "" : "s")

	#if CURSES_ENABLED
	# define print_dot() if (output_format == mon_output_console) { \
	printw("."); \
	clrtoeol(); \
	refresh(); \
	} else { \
	fprintf(stdout, "."); \
	}
	#else
	# define print_dot() fprintf(stdout, ".");
	#endif

	#if CURSES_ENABLED
	# define print_as(fmt, args...) if (output_format == mon_output_console) { \
	printw(fmt, ##args); \
	clrtoeol(); \
	refresh(); \
	} else { \
	fprintf(stdout, fmt, ##args); \
	}
	#else
	# define print_as(fmt, args...) fprintf(stdout, fmt, ##args);
	#endif

	static void
	blank_screen(void)
	{
	#if CURSES_ENABLED
	int lpc = 0;

	for (lpc = 0; lpc < LINES; lpc++) {
	move(lpc, 0);
	clrtoeol();
	}
	move(0, 0);
	refresh();
	#endif
	}

	static gboolean
	mon_timer_popped(gpointer data)
	{
	int rc = pcmk_ok;

	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	clear();
	refresh();
	}
	#endif

	if (timer_id > 0) {
	g_source_remove(timer_id);
	timer_id = 0;
	}

	print_as("Reconnecting...\n");
	rc = cib_connect(TRUE);

	if (rc != pcmk_ok) {
	timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
	}
	return FALSE;
	}

	static void
	mon_cib_connection_destroy(gpointer user_data)
	{
	print_as("Connection to the cluster-daemons terminated\n");
	if (refresh_timer != NULL) {
	/* we'll trigger a refresh after reconnect */
	mainloop_timer_stop(refresh_timer);
	}
	if (timer_id) {
	/* we'll trigger a new reconnect-timeout at the end */
	g_source_remove(timer_id);
	timer_id = 0;
	}
	if (st) {
	/* the client API won't properly reconnect notifications
	* if they are still in the table - so remove them
	*/
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY);
	if (st->state != stonith_disconnected) {
	st->cmds->disconnect(st);
	}
	}
	if (cib) {
	cib->cmds->signoff(cib);
	timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
	}
	return;
	}

	/*
	* Mainloop signal handler.
	*/
	static void
	mon_shutdown(int nsig)
	{
	clean_up(CRM_EX_OK);
	}

	#if CURSES_ENABLED
	static sighandler_t ncurses_winch_handler;

	static void
	mon_winresize(int nsig)
	{
	static int not_done;
	int lines = 0, cols = 0;

	if (!not_done++) {
	if (ncurses_winch_handler)
	/* the original ncurses WINCH signal handler does the
	* magic of retrieving the new window size;
	* otherwise, we'd have to use ioctl or tgetent */
	(*ncurses_winch_handler) (SIGWINCH);
	getmaxyx(stdscr, lines, cols);
	resizeterm(lines, cols);
	mainloop_set_trigger(refresh_trigger);
	}
	not_done--;
	}
	#endif

	static int
	cib_connect(gboolean full)
	{
	int rc = pcmk_ok;
	static gboolean need_pass = TRUE;

	CRM_CHECK(cib != NULL, return -EINVAL);

	if (getenv("CIB_passwd") != NULL) {
	need_pass = FALSE;
	}

	if ((fence_connect) && (st == NULL)) {
	st = stonith_api_new();
	}

	if ((fence_connect) && (st->state == stonith_disconnected)) {
	crm_trace("Connecting to stonith");
	rc = st->cmds->connect(st, crm_system_name, NULL);
	if (rc == pcmk_ok) {
	crm_trace("Setting up stonith callbacks");
	if (watch_fencing) {
	st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
	mon_st_callback_event);
	st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
	} else {
	st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
	mon_st_callback_display);
	st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
	}
	}
	}

	if (cib->state != cib_connected_query && cib->state != cib_connected_command) {
	crm_trace("Connecting to the CIB");
	if ((output_format == mon_output_console) && need_pass && (cib->variant == cib_remote)) {
	need_pass = FALSE;
	print_as("Password:");
	}

	rc = cib->cmds->signon(cib, crm_system_name, cib_query);

	if (rc != pcmk_ok) {
	return rc;
	}

	rc = cib->cmds->query(cib, NULL, &current_cib, cib_scope_local \| cib_sync_call);
	if (rc == pcmk_ok) {
	mon_refresh_display(NULL);
	}

	if (rc == pcmk_ok && full) {
	if (rc == pcmk_ok) {
	rc = cib->cmds->set_connection_dnotify(cib, mon_cib_connection_destroy);
	if (rc == -EPROTONOSUPPORT) {
	print_as
	("Notification setup not supported, won't be able to reconnect after failure");
	if (output_format == mon_output_console) {
	sleep(2);
	}
	rc = pcmk_ok;
	}

	}

	if (rc == pcmk_ok) {
	cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
	rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
	}

	if (rc != pcmk_ok) {
	print_as("Notification setup failed, could not monitor CIB actions");
	if (output_format == mon_output_console) {
	sleep(2);
	}
	clean_up_connections();
	}
	}
	}
	return rc;
	}

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},
	{"quiet", 0, 0, 'Q', "\tDisplay only essential output" },

	{"-spacer-", 1, 0, '-', "\nModes (mutually exclusive):"},
	{"as-html", 1, 0, 'h', "\tWrite cluster status to the named html file"},
	{"as-xml", 0, 0, 'X', "\t\tWrite cluster status as xml to stdout. This will enable one-shot mode."},
	{"web-cgi", 0, 0, 'w', "\t\tWeb mode with output suitable for CGI (preselected when run as *.cgi)"},
	{"simple-status", 0, 0, 's', "\tDisplay the cluster status once as a simple one line output (suitable for nagios)"},
	{"-spacer-", 1, 0, '-', "\nDisplay Options:"},
	{"group-by-node", 0, 0, 'n', "\tGroup resources by node" },
	{"inactive", 0, 0, 'r', "\t\tDisplay inactive resources" },
	{"failcounts", 0, 0, 'f', "\tDisplay resource fail counts"},
	{"operations", 0, 0, 'o', "\tDisplay resource operation history" },
	{"timing-details", 0, 0, 't', "\tDisplay resource operation history with timing details" },
	{"tickets", 0, 0, 'c', "\t\tDisplay cluster tickets"},
	{"watch-fencing", 0, 0, 'W', "\tListen for fencing events. For use with --external-agent"},
	{"fence-history", 2, 0, 'm', "Show fence history\n"
	"\t\t\t\t\t0=off, 1=failures and pending (default without option),\n"
	"\t\t\t\t\t2=add successes (default without value for option),\n"
	"\t\t\t\t\t3=show full history without reduction to most recent of each flavor"},
	{"neg-locations", 2, 0, 'L', "Display negative location constraints [optionally filtered by id prefix]"},
	{"show-node-attributes", 0, 0, 'A', "Display node attributes" },
	{"hide-headers", 0, 0, 'D', "\tHide all headers" },
	{"show-detail", 0, 0, 'R', "\tShow more details (node IDs, individual clone instances)" },
	{"brief", 0, 0, 'b', "\t\tBrief output" },
	{"pending", 0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled", pcmk_option_hidden},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{"interval", 1, 0, 'i', "\tUpdate frequency in seconds" },
	{"one-shot", 0, 0, '1', "\t\tDisplay the cluster status once on the console and exit"},
	{"disable-ncurses",0, 0, 'N', "\tDisable the use of ncurses", !CURSES_ENABLED},
	{"daemonize", 0, 0, 'd', "\tRun in the background as a daemon"},
	{"pid-file", 1, 0, 'p', "\t(Advanced) Daemon pid file location"},
	{"external-agent", 1, 0, 'E', "A program to run when resource operations take place."},
	{"external-recipient",1, 0, 'e', "A recipient for your program (assuming you want the program to send something to someone)."},


	{"xml-file", 1, 0, 'x', NULL, pcmk_option_hidden},

	{"-spacer-", 1, 0, '-', "\nExamples:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "Display the cluster status on the console with updates as they occur:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_mon", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Display the cluster status on the console just once then exit:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_mon -1", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Display your cluster status, group resources by node, and include inactive resources in the list:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_mon --group-by-node --inactive", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_mon --daemonize --as-html /path/to/docroot/filename.html", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Start crm_mon and export the current cluster status as xml to stdout, then exit.:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_mon --as-xml", pcmk_option_example},

	{NULL, 0, 0, 0}
	};
	/* INDENT-ON */

	#if CURSES_ENABLED
	static const char *
	get_option_desc(char c)
	{
	int lpc;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {

	if (long_options[lpc].name[0] == '-')
	continue;

	if (long_options[lpc].val == c) {
	static char *buf = NULL;
	const char *rv;
	char *nl;

	/* chop off tabs and cut at newline */
	free(buf); /* free string from last usage */
	buf = strdup(long_options[lpc].desc);
	rv = buf; /* make a copy to keep buf pointer unaltered
	for freeing when we come by next time.
	Like this the result stays valid until
	the next call.
	*/
	while(isspace(rv[0])) {
	rv++;
	}
	nl = strchr(rv, '\n');
	if (nl) {
	*nl = '\0';
	}
	return rv;
	}
	}

	return NULL;
	}

	#define print_option_help(option, condition) \
	print_as("%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));

	static gboolean
	detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer unused)
	{
	int c;
	gboolean config_mode = FALSE;

	while (1) {

	/* Get user input */
	c = getchar();

	switch (c) {
	case 'm':
	if (!fence_history_level) {
	fence_history = TRUE;
	fence_connect = TRUE;
	if (st == NULL) {
	mon_cib_connection_destroy(NULL);
	}
	}
	show ^= mon_show_fence_history;
	break;
	case 'c':
	show ^= mon_show_tickets;
	break;
	case 'f':
	show ^= mon_show_failcounts;
	break;
	case 'n':
	group_by_node = ! group_by_node;
	break;
	case 'o':
	show ^= mon_show_operations;
	if ((show & mon_show_operations) == 0) {
	print_timing = 0;
	}
	break;
	case 'r':
	inactive_resources = ! inactive_resources;
	break;
	case 'R':
	print_clone_detail = ! print_clone_detail;
	break;
	case 't':
	print_timing = ! print_timing;
	if (print_timing) {
	show \|= mon_show_operations;
	}
	break;
	case 'A':
	show ^= mon_show_attributes;
	break;
	case 'L':
	show ^= mon_show_bans;
	break;
	case 'D':
	/* If any header is shown, clear them all, otherwise set them all */
	if (show & mon_show_headers) {
	show &= ~mon_show_headers;
	} else {
	show \|= mon_show_headers;
	}
	break;
	case 'b':
	print_brief = ! print_brief;
	break;
	case 'j':
	print_pending = ! print_pending;
	break;
	case '?':
	config_mode = TRUE;
	break;
	default:
	goto refresh;
	}

	if (!config_mode)
	goto refresh;

	blank_screen();

	print_as("Display option change mode\n");
	print_as("\n");
	print_option_help('c', show & mon_show_tickets);
	print_option_help('f', show & mon_show_failcounts);
	print_option_help('n', group_by_node);
	print_option_help('o', show & mon_show_operations);
	print_option_help('r', inactive_resources);
	print_option_help('t', print_timing);
	print_option_help('A', show & mon_show_attributes);
	print_option_help('L', show & mon_show_bans);
	print_option_help('D', (show & mon_show_headers) == 0);
	print_option_help('R', print_clone_detail);
	print_option_help('b', print_brief);
	print_option_help('j', print_pending);
	print_option_help('m', (show & mon_show_fence_history));
	print_as("\n");
	print_as("Toggle fields via field letter, type any other key to return");
	}

	refresh:
	mon_refresh_display(NULL);
	return TRUE;
	}
	#endif

	-// Basically crm_signal(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
	+// Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
	static void
	avoid_zombies()
	{
	struct sigaction sa;

	memset(&sa, 0, sizeof(struct sigaction));
	if (sigemptyset(&sa.sa_mask) < 0) {
	crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
	return;
	}
	sa.sa_handler = SIG_IGN;
	sa.sa_flags = SA_RESTART\|SA_NOCLDWAIT;
	if (sigaction(SIGCHLD, &sa, NULL) < 0) {
	crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
	}
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int argerr = 0;
	int option_index = 0;
	int rc = pcmk_ok;

	pid_file = strdup("/tmp/ClusterMon.pid");
	crm_log_cli_init("crm_mon");
	crm_set_options(NULL, "mode [options]", long_options,
	"Provides a summary of cluster's current state."
	"\n\nOutputs varying levels of detail in a number of different formats.\n");

	// Avoid needing to wait for subprocesses forked for -E/--external-agent
	avoid_zombies();

	if (crm_ends_with_ext(argv[0], ".cgi") == TRUE) {
	output_format = mon_output_cgi;
	one_shot = TRUE;
	}

	/* to enable stonith-connection when called via some application like pcs
	* set environment-variable FENCE_HISTORY to desired level
	* so you don't have to modify this application
	*/
	/* fence_history_level = crm_atoi(getenv("FENCE_HISTORY"), "0"); */

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'Q':
	show &= ~mon_show_times;
	break;
	case 'i':
	reconnect_msec = crm_get_msec(optarg);
	break;
	case 'n':
	group_by_node = TRUE;
	break;
	case 'r':
	inactive_resources = TRUE;
	break;
	case 'W':
	watch_fencing = TRUE;
	fence_connect = TRUE;
	break;
	case 'm':
	fence_history_level = crm_atoi(optarg, "2");
	break;
	case 'd':
	daemonize = TRUE;
	break;
	case 't':
	print_timing = TRUE;
	show \|= mon_show_operations;
	break;
	case 'o':
	show \|= mon_show_operations;
	break;
	case 'f':
	show \|= mon_show_failcounts;
	break;
	case 'A':
	show \|= mon_show_attributes;
	break;
	case 'L':
	show \|= mon_show_bans;
	print_neg_location_prefix = optarg? optarg : "";
	break;
	case 'D':
	show &= ~mon_show_headers;
	break;
	case 'b':
	print_brief = TRUE;
	break;
	case 'j':
	print_pending = TRUE;
	break;
	case 'R':
	print_clone_detail = TRUE;
	break;
	case 'c':
	show \|= mon_show_tickets;
	break;
	case 'p':
	free(pid_file);
	if(optarg == NULL) {
	crm_help(flag, CRM_EX_USAGE);
	}
	pid_file = strdup(optarg);
	break;
	case 'x':
	if(optarg == NULL) {
	crm_help(flag, CRM_EX_USAGE);
	}
	setenv("CIB_file", optarg, 1);
	one_shot = TRUE;
	break;
	case 'h':
	if(optarg == NULL) {
	crm_help(flag, CRM_EX_USAGE);
	}
	argerr += (output_format != mon_output_console);
	output_format = mon_output_html;
	output_filename = strdup(optarg);
	umask(S_IWGRP \| S_IWOTH);
	break;
	case 'X':
	argerr += (output_format != mon_output_console);
	output_format = mon_output_xml;
	one_shot = TRUE;
	break;
	case 'w':
	/* do not allow argv[0] and argv[1...] redundancy */
	argerr += (output_format != mon_output_console);
	output_format = mon_output_cgi;
	one_shot = TRUE;
	break;
	case 's':
	argerr += (output_format != mon_output_console);
	output_format = mon_output_monitor;
	one_shot = TRUE;
	break;
	case 'E':
	external_agent = optarg;
	break;
	case 'e':
	external_recipient = optarg;
	break;
	case '1':
	one_shot = TRUE;
	break;
	case 'N':
	if (output_format == mon_output_console) {
	output_format = mon_output_plain;
	}
	break;
	case '$':
	case '?':
	crm_help(flag, CRM_EX_OK);
	break;
	default:
	printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (watch_fencing) {
	/* don't moan as fence_history_level == 1 is default */
	fence_history_level = 0;
	}

	/* create the cib-object early to be able to do further
	* decisions based on the cib-source
	*/
	cib = cib_new();

	if (cib == NULL) {
	rc = -EINVAL;
	} else {
	switch (cib->variant) {

	case cib_native:
	/* cib & fencing - everything available */
	break;

	case cib_file:
	/* Don't try to connect to fencing as we
	* either don't have a running cluster or
	* the fencing-information would possibly
	* not match the cib data from a file.
	* As we don't expect cib-updates coming
	* in enforce one-shot. */
	fence_history_level = 0;
	one_shot = TRUE;
	break;

	case cib_remote:
	/* updates coming in but no fencing */
	fence_history_level = 0;
	break;

	case cib_undefined:
	case cib_database:
	default:
	/* something is odd */
	rc = -EINVAL;
	crm_err("Invalid cib-source");
	break;
	}
	}

	switch (fence_history_level) {
	case 3:
	fence_full_history = TRUE;
	/* fall through to next lower level */
	case 2:
	show \|= mon_show_fence_history;
	/* fall through to next lower level */
	case 1:
	fence_history = TRUE;
	fence_connect = TRUE;
	break;
	default:
	break;
	}

	/* Extra sanity checks when in CGI mode */
	if (output_format == mon_output_cgi) {
	argerr += (optind < argc);
	argerr += (output_filename != NULL);
	argerr += ((cib) && (cib->variant == cib_file));
	argerr += (external_agent != NULL);
	argerr += (daemonize == TRUE); /* paranoia */

	} else if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}

	if (argerr) {
	return clean_up(CRM_EX_USAGE);
	}

	/* XML output always prints everything */
	if (output_format == mon_output_xml) {
	show = mon_show_all;
	print_timing = TRUE;
	}

	if (one_shot) {
	if (output_format == mon_output_console) {
	output_format = mon_output_plain;
	}

	} else if (daemonize) {
	if ((output_format == mon_output_console) \|\| (output_format == mon_output_plain)) {
	output_format = mon_output_none;
	}
	crm_enable_stderr(FALSE);

	if ((output_format != mon_output_html)
	&& !external_agent) {
	printf ("Looks like you forgot to specify one or more of: "
	"--as-html, --external-agent\n");
	return clean_up(CRM_EX_USAGE);
	}

	if (cib) {
	/* to be on the safe side don't have cib-object around
	* when we are forking
	*/
	cib_delete(cib);
	cib = NULL;
	crm_make_daemon(crm_system_name, TRUE, pid_file);
	cib = cib_new();
	if (cib == NULL) {
	rc = -EINVAL;
	}
	/* otherwise assume we've got the same cib-object we've just destroyed
	* in our parent
	*/
	}


	} else if (output_format == mon_output_console) {
	#if CURSES_ENABLED
	initscr();
	cbreak();
	noecho();
	crm_enable_stderr(FALSE);
	curses_console_initialized = TRUE;
	#else
	one_shot = TRUE;
	output_format = mon_output_plain;
	printf("Defaulting to one-shot mode\n");
	printf("You need to have curses available at compile time to enable console mode\n");
	#endif
	}

	crm_info("Starting %s", crm_system_name);

	if (cib) {

	do {
	if (!one_shot) {
	print_as("Waiting until cluster is available on this node ...\n");
	}
	rc = cib_connect(!one_shot);

	if (one_shot) {
	break;

	} else if (rc != pcmk_ok) {
	sleep(reconnect_msec / 1000);
	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	clear();
	refresh();
	}
	#endif
	} else {
	if (output_format == mon_output_html) {
	print_as("Writing html to %s ...\n", output_filename);
	}
	}

	} while (rc == -ENOTCONN);
	}

	if (rc != pcmk_ok) {
	if (output_format == mon_output_monitor) {
	printf("CLUSTER CRIT: Connection to cluster failed: %s\n",
	pcmk_strerror(rc));
	return clean_up(MON_STATUS_CRIT);
	} else {
	if (rc == -ENOTCONN) {
	print_as("\nError: cluster is not available on this node\n");
	} else {
	print_as("\nConnection to cluster failed: %s\n",
	pcmk_strerror(rc));
	}
	}
	if (output_format == mon_output_console) {
	sleep(2);
	}
	return clean_up(crm_errno2exit(rc));
	}

	if (one_shot) {
	return clean_up(CRM_EX_OK);
	}

	mainloop = g_main_loop_new(NULL, FALSE);

	mainloop_add_signal(SIGTERM, mon_shutdown);
	mainloop_add_signal(SIGINT, mon_shutdown);
	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	- ncurses_winch_handler = crm_signal(SIGWINCH, mon_winresize);
	+ ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
	if (ncurses_winch_handler == SIG_DFL \|\|
	ncurses_winch_handler == SIG_IGN \|\| ncurses_winch_handler == SIG_ERR)
	ncurses_winch_handler = NULL;
	g_io_add_watch(g_io_channel_unix_new(STDIN_FILENO), G_IO_IN, detect_user_input, NULL);
	}
	#endif
	refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);

	g_main_loop_run(mainloop);
	g_main_loop_unref(mainloop);

	crm_info("Exiting %s", crm_system_name);

	return clean_up(CRM_EX_OK);
	}

	#define mon_warn(fmt...) do { \
	if (!has_warnings) { \
	print_as("CLUSTER WARN:"); \
	} else { \
	print_as(","); \
	} \
	print_as(fmt); \
	has_warnings = TRUE; \
	} while(0)

	static int
	count_resources(pe_working_set_t * data_set, resource_t * rsc)
	{
	int count = 0;
	GListPtr gIter = NULL;

	if (rsc == NULL) {
	gIter = data_set->resources;
	} else if (rsc->children) {
	gIter = rsc->children;
	} else {
	return is_not_set(rsc->flags, pe_rsc_orphan);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	count += count_resources(data_set, gIter->data);
	}
	return count;
	}

	/*!
	* \internal
	* \brief Print one-line status suitable for use with monitoring software
	*
	* \param[in] data_set Working set of CIB state
	* \param[in] history List of stonith actions
	*
	* \note This function's output (and the return code when the program exits)
	* should conform to https://www.monitoring-plugins.org/doc/guidelines.html
	*/
	static void
	print_simple_status(pe_working_set_t * data_set,
	stonith_history_t *history)
	{
	GListPtr gIter = NULL;
	int nodes_online = 0;
	int nodes_standby = 0;
	int nodes_maintenance = 0;

	if (data_set->dc_node == NULL) {
	mon_warn(" No DC");
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (node->details->standby && node->details->online) {
	nodes_standby++;
	} else if (node->details->maintenance && node->details->online) {
	nodes_maintenance++;
	} else if (node->details->online) {
	nodes_online++;
	} else {
	mon_warn(" offline node: %s", node->details->uname);
	}
	}

	if (!has_warnings) {
	int nresources = count_resources(data_set, NULL);

	print_as("CLUSTER OK: %d node%s online", nodes_online, s_if_plural(nodes_online));
	if (nodes_standby > 0) {
	print_as(", %d standby node%s", nodes_standby, s_if_plural(nodes_standby));
	}
	if (nodes_maintenance > 0) {
	print_as(", %d maintenance node%s", nodes_maintenance, s_if_plural(nodes_maintenance));
	}
	print_as(", %d resource%s configured", nresources, s_if_plural(nresources));
	}

	print_as("\n");
	}

	/*!
	* \internal
	* \brief Print a [name]=[value][units] pair, optionally using time string
	*
	* \param[in] stream File stream to display output to
	* \param[in] name Name to display
	* \param[in] value Value to display (or NULL to convert time instead)
	* \param[in] units Units to display (or NULL for no units)
	* \param[in] epoch_time Epoch time to convert if value is NULL
	*/
	static void
	print_nvpair(FILE stream, const char name, const char *value,
	const char *units, time_t epoch_time)
	{
	/* print name= */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" %s=", name);
	break;

	case mon_output_html:
	case mon_output_cgi:
	case mon_output_xml:
	fprintf(stream, " %s=", name);
	break;

	default:
	break;
	}

	/* If we have a value (and optionally units), print it */
	if (value) {
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("%s%s", value, (units? units : ""));
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, "%s%s", value, (units? units : ""));
	break;

	case mon_output_xml:
	fprintf(stream, "\"%s%s\"", value, (units? units : ""));
	break;

	default:
	break;
	}

	/* Otherwise print user-friendly time string */
	} else {
	static char empty_str[] = "";
	char c, date_str = asctime(localtime(&epoch_time));

	for (c = (date_str != NULL) ? date_str : empty_str; *c != '\0'; ++c) {
	if (*c == '\n') {
	*c = '\0';
	break;
	}
	}
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("'%s'", date_str);
	break;

	case mon_output_html:
	case mon_output_cgi:
	case mon_output_xml:
	fprintf(stream, "\"%s\"", date_str);
	break;

	default:
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Print whatever is needed to start a node section
	*
	* \param[in] stream File stream to display output to
	* \param[in] node Node to print
	*/
	static void
	print_node_start(FILE stream, node_t node)
	{
	char *node_name;

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	node_name = get_node_display_name(node);
	print_as("* Node %s:\n", node_name);
	free(node_name);
	break;

	case mon_output_html:
	case mon_output_cgi:
	node_name = get_node_display_name(node);
	fprintf(stream, " <h3>Node: %s</h3>\n <ul>\n", node_name);
	free(node_name);
	break;

	case mon_output_xml:
	fprintf(stream, " <node name=\"%s\">\n", node->details->uname);
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print whatever is needed to end a node section
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_node_end(FILE *stream)
	{
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </node>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print resources section heading appropriate to options
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_resources_heading(FILE *stream)
	{
	const char *heading;

	if (group_by_node) {

	/* Active resources have already been printed by node */
	heading = (inactive_resources? "Inactive resources" : NULL);

	} else if (inactive_resources) {
	heading = "Full list of resources";

	} else {
	heading = "Active resources";
	}

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\n%s:\n\n", heading);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>%s</h2>\n", heading);
	break;

	case mon_output_xml:
	fprintf(stream, " <resources>\n");
	break;

	default:
	break;
	}

	}

	/*!
	* \internal
	* \brief Print whatever resource section closing is appropriate
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_resources_closing(FILE *stream, gboolean printed_heading)
	{
	const char *heading;

	/* What type of resources we did or did not display */
	if (group_by_node) {
	heading = "inactive ";
	} else if (inactive_resources) {
	heading = "";
	} else {
	heading = "active ";
	}

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	if (!printed_heading) {
	print_as("\nNo %sresources\n\n", heading);
	}
	break;

	case mon_output_html:
	case mon_output_cgi:
	if (!printed_heading) {
	fprintf(stream, " <hr />\n <h2>No %sresources</h2>\n", heading);
	}
	break;

	case mon_output_xml:
	fprintf(stream, " %s\n",
	(printed_heading? "</resources>" : "<resources/>"));
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print whatever resource section(s) are appropriate
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Cluster state to display
	* \param[in] print_opts Bitmask of pe_print_options
	*/
	static void
	print_resources(FILE stream, pe_working_set_t data_set, int print_opts)
	{
	GListPtr rsc_iter;
	const char *prefix = NULL;
	gboolean printed_heading = FALSE;
	gboolean brief_output = print_brief;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (group_by_node && !inactive_resources) {
	return;
	}

	/* XML uses an indent, and ignores brief option for resources */
	if (output_format == mon_output_xml) {
	prefix = " ";
	brief_output = FALSE;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (brief_output && !group_by_node) {
	print_resources_heading(stream);
	printed_heading = TRUE;
	print_rscs_brief(data_set->resources, NULL, print_opts, stream,
	inactive_resources);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	resource_t rsc = (resource_t ) rsc_iter->data;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (group_by_node) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (brief_output && (rsc->variant == pe_native)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !inactive_resources) {
	continue;
	}

	/* Print this resource */
	if (printed_heading == FALSE) {
	print_resources_heading(stream);
	printed_heading = TRUE;
	}
	rsc->fns->print(rsc, prefix, print_opts, stream);
	}

	print_resources_closing(stream, printed_heading);
	}

	/*!
	* \internal
	* \brief Print heading for resource history
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Current state of CIB
	* \param[in] node Node that ran this resource
	* \param[in] rsc Resource to print
	* \param[in] rsc_id ID of resource to print
	* \param[in] all Whether to print every resource or just failed ones
	*/
	static void
	print_rsc_history_start(FILE stream, pe_working_set_t data_set, node_t *node,
	resource_t rsc, const char rsc_id, gboolean all)
	{
	time_t last_failure = 0;
	int failcount = rsc?
	pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
	NULL, data_set)
	: 0;

	if (!all && !failcount && (last_failure <= 0)) {
	return;
	}

	/* Print resource ID */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" %s:", rsc_id);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <li>%s:", rsc_id);
	break;

	case mon_output_xml:
	fprintf(stream, " <resource_history id=\"%s\"", rsc_id);
	break;

	default:
	break;
	}

	/* If resource is an orphan, that's all we can say about it */
	if (rsc == NULL) {
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" orphan");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " orphan");
	break;

	case mon_output_xml:
	fprintf(stream, " orphan=\"true\"");
	break;

	default:
	break;
	}

	/* If resource is not an orphan, print some details */
	} else if (all \|\| failcount \|\| (last_failure > 0)) {

	/* Print migration threshold */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" migration-threshold=%d", rsc->migration_threshold);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " migration-threshold=%d", rsc->migration_threshold);
	break;

	case mon_output_xml:
	fprintf(stream, " orphan=\"false\" migration-threshold=\"%d\"",
	rsc->migration_threshold);
	break;

	default:
	break;
	}

	/* Print fail count if any */
	if (failcount > 0) {
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
	break;

	case mon_output_xml:
	fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=\"%d\"",
	failcount);
	break;

	default:
	break;
	}
	}

	/* Print last failure time if any */
	if (last_failure > 0) {
	print_nvpair(stream, CRM_LAST_FAILURE_PREFIX, NULL, NULL,
	last_failure);
	}
	}

	/* End the heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, "\n <ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, ">\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print closing for resource history
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_rsc_history_end(FILE *stream)
	{
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n </li>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </resource_history>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print operation history
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Current state of CIB
	* \param[in] node Node this operation is for
	* \param[in] xml_op Root of XML tree describing this operation
	* \param[in] task Task parsed from this operation's XML
	* \param[in] interval_ms_s Interval parsed from this operation's XML
	* \param[in] rc Return code parsed from this operation's XML
	*/
	static void
	print_op_history(FILE stream, pe_working_set_t data_set, node_t *node,
	xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc)
	{
	const char *value = NULL;
	const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);

	/* Begin the operation description */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" + (%s) %s:", call, task);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <li>(%s) %s:", call, task);
	break;

	case mon_output_xml:
	fprintf(stream, " <operation_history call=\"%s\" task=\"%s\"",
	call, task);
	break;

	default:
	break;
	}

	/* Add name=value pairs as appropriate */
	if (interval_ms_s && safe_str_neq(interval_ms_s, "0")) {
	print_nvpair(stream, "interval", interval_ms_s, "ms", 0);
	}
	if (print_timing) {
	int int_value;
	const char *attr;

	attr = XML_RSC_OP_LAST_CHANGE;
	value = crm_element_value(xml_op, attr);
	if (value) {
	int_value = crm_parse_int(value, NULL);
	if (int_value > 0) {
	print_nvpair(stream, attr, NULL, NULL, int_value);
	}
	}

	attr = XML_RSC_OP_LAST_RUN;
	value = crm_element_value(xml_op, attr);
	if (value) {
	int_value = crm_parse_int(value, NULL);
	if (int_value > 0) {
	print_nvpair(stream, attr, NULL, NULL, int_value);
	}
	}

	attr = XML_RSC_OP_T_EXEC;
	value = crm_element_value(xml_op, attr);
	if (value) {
	print_nvpair(stream, attr, value, "ms", 0);
	}

	attr = XML_RSC_OP_T_QUEUE;
	value = crm_element_value(xml_op, attr);
	if (value) {
	print_nvpair(stream, attr, value, "ms", 0);
	}
	}

	/* End the operation description */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" rc=%d (%s)\n", rc, services_ocf_exitcode_str(rc));
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " rc=%d (%s)</li>\n", rc, services_ocf_exitcode_str(rc));
	break;

	case mon_output_xml:
	fprintf(stream, " rc=\"%d\" rc_text=\"%s\" />\n", rc, services_ocf_exitcode_str(rc));
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print resource operation/failure history
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Current state of CIB
	* \param[in] node Node that ran this resource
	* \param[in] rsc_entry Root of XML tree describing resource status
	* \param[in] operations Whether to print operations or just failcounts
	*/
	static void
	print_rsc_history(FILE stream, pe_working_set_t data_set, node_t *node,
	xmlNode *rsc_entry, gboolean operations)
	{
	GListPtr gIter = NULL;
	GListPtr op_list = NULL;
	gboolean printed = FALSE;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
	xmlNode *rsc_op = NULL;

	/* If we're not showing operations, just print the resource failure summary */
	if (operations == FALSE) {
	print_rsc_history_start(stream, data_set, node, rsc, rsc_id, FALSE);
	print_rsc_history_end(stream);
	return;
	}

	/* Create a list of this resource's operations */
	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}
	op_list = g_list_sort(op_list, sort_op_by_callid);

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(xml_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	int rc = crm_parse_int(op_rc, "0");

	/* Display 0-interval monitors as "probe" */
	if (safe_str_eq(task, CRMD_ACTION_STATUS)
	&& ((interval_ms_s == NULL) \|\| safe_str_eq(interval_ms_s, "0"))) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (safe_str_eq(task, CRMD_ACTION_NOTIFY) \|\| (safe_str_eq(task, "probe") && (rc == 7))) {
	continue;
	}

	/* If this is the first printed operation, print heading for resource */
	if (printed == FALSE) {
	printed = TRUE;
	print_rsc_history_start(stream, data_set, node, rsc, rsc_id, TRUE);
	}

	/* Print the operation */
	print_op_history(stream, data_set, node, xml_op, task, interval_ms_s,
	rc);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	/* If we printed anything, close the resource */
	if (printed) {
	print_rsc_history_end(stream);
	}
	}

	/*!
	* \internal
	* \brief Print node operation/failure history
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Current state of CIB
	* \param[in] node_state Root of XML tree describing node status
	* \param[in] operations Whether to print operations or just failcounts
	*/
	static void
	print_node_history(FILE stream, pe_working_set_t data_set,
	xmlNode *node_state, gboolean operations)
	{
	node_t *node = pe_find_node_id(data_set->nodes, ID(node_state));
	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;

	if (node && node->details && node->details->online) {
	print_node_start(stream, node);

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = __xml_first_child(lrm_rsc); rsc_entry != NULL;
	rsc_entry = __xml_next(rsc_entry)) {

	if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
	print_rsc_history(stream, data_set, node, rsc_entry, operations);
	}
	}

	print_node_end(stream);
	}
	}

	/*!
	* \internal
	* \brief Print extended information about an attribute if appropriate
	*
	* \param[in] data_set Working set of CIB state
	*
	* \return TRUE if extended information was printed, FALSE otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static gboolean
	print_attr_msg(FILE stream, node_t node, GListPtr rsc_list, const char attrname, const char attrvalue)
	{
	GListPtr gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, "type");

	if (rsc->children != NULL) {
	if (print_attr_msg(stream, node, rsc->children, attrname, attrvalue)) {
	return TRUE;
	}
	}

	if (safe_str_eq(type, "ping") \|\| safe_str_eq(type, "pingd")) {
	const char *name = g_hash_table_lookup(rsc->parameters, "name");

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (safe_str_eq(name, attrname)) {
	int host_list_num = 0;
	int expected_score = 0;
	int value = crm_parse_int(attrvalue, "0");
	const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
	const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");

	if(hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	/* pingd multiplier is the same as the default value. */
	expected_score = host_list_num * crm_parse_int(multiplier, "1");

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	if (value <= 0) {
	print_as("\t: Connectivity is lost");
	} else if (value < expected_score) {
	print_as("\t: Connectivity is degraded (Expected=%d)", expected_score);
	}
	break;

	case mon_output_html:
	case mon_output_cgi:
	if (value <= 0) {
	fprintf(stream, " <b>(connectivity is lost)</b>");
	} else if (value < expected_score) {
	fprintf(stream, " <b>(connectivity is degraded -- expected %d)</b>",
	expected_score);
	}
	break;

	case mon_output_xml:
	fprintf(stream, " expected=\"%d\"", expected_score);
	break;

	default:
	break;
	}
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	static void
	create_attr_list(gpointer name, gpointer value, gpointer data)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return;
	}
	}

	attr_list = g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	/* structure for passing multiple user data to g_list_foreach() */
	struct mon_attr_data {
	FILE *stream;
	node_t *node;
	};

	static void
	print_node_attribute(gpointer name, gpointer user_data)
	{
	const char *value = NULL;
	struct mon_attr_data data = (struct mon_attr_data ) user_data;

	value = pe_node_attribute_raw(data->node, name);

	/* Print attribute name and value */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(" + %-32s\t: %-10s", (char *)name, value);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(data->stream, " <li>%s: %s",
	(char *)name, value);
	break;

	case mon_output_xml:
	fprintf(data->stream,
	" <attribute name=\"%s\" value=\"%s\"",
	(char *)name, value);
	break;

	default:
	break;
	}

	/* Print extended information if appropriate */
	print_attr_msg(data->stream, data->node, data->node->details->running_rsc,
	name, value);

	/* Close out the attribute */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(data->stream, "</li>\n");
	break;

	case mon_output_xml:
	fprintf(data->stream, " />\n");
	break;

	default:
	break;
	}
	}

	static void
	print_node_summary(FILE stream, pe_working_set_t data_set, gboolean operations)
	{
	xmlNode *node_state = NULL;
	xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);

	/* Print heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	if (operations) {
	print_as("\nOperations:\n");
	} else {
	print_as("\nMigration Summary:\n");
	}
	break;

	case mon_output_html:
	case mon_output_cgi:
	if (operations) {
	fprintf(stream, " <hr />\n <h2>Operations</h2>\n");
	} else {
	fprintf(stream, " <hr />\n <h2>Migration Summary</h2>\n");
	}
	break;

	case mon_output_xml:
	fprintf(stream, " <node_history>\n");
	break;

	default:
	break;
	}

	/* Print each node in the CIB status */
	for (node_state = __xml_first_child(cib_status); node_state != NULL;
	node_state = __xml_next(node_state)) {
	if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
	print_node_history(stream, data_set, node_state, operations);
	}
	}

	/* Close section */
	switch (output_format) {
	case mon_output_xml:
	fprintf(stream, " </node_history>\n");
	break;

	default:
	break;
	}
	}

	static void
	print_ticket(gpointer name, gpointer value, gpointer data)
	{
	ticket_t ticket = (ticket_t ) value;
	FILE stream = (FILE ) data;

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("* %s:\t%s%s", ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""));
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <li>%s: %s%s", ticket->id,
	(ticket->granted? "granted" : "revoked"),
	(ticket->standby? " [standby]" : ""));
	break;

	case mon_output_xml:
	fprintf(stream, " <ticket id=\"%s\" status=\"%s\" standby=\"%s\"",
	ticket->id, (ticket->granted? "granted" : "revoked"),
	(ticket->standby? "true" : "false"));
	break;

	default:
	break;
	}
	if (ticket->last_granted > -1) {
	print_nvpair(stdout, "last-granted", NULL, NULL, ticket->last_granted);
	}
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, "</li>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " />\n");
	break;

	default:
	break;
	}
	}

	static void
	print_cluster_tickets(FILE stream, pe_working_set_t data_set)
	{
	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nTickets:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Tickets</h2>\n <ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <tickets>\n");
	break;

	default:
	break;
	}

	/* Print each ticket */
	g_hash_table_foreach(data_set->tickets, print_ticket, stream);

	/* Close section */
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </tickets>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Return human-friendly string representing node name
	*
	* The returned string will be in the format
	* uname[@hostUname] [(nodeID)]
	* "@hostUname" will be printed if the node is a guest node.
	* "(nodeID)" will be printed if the node ID is different from the node uname,
	* and detailed output has been requested.
	*
	* \param[in] node Node to represent
	* \return Newly allocated string with representation of node name
	* \note It is the caller's responsibility to free the result with free().
	*/
	static char *
	get_node_display_name(node_t *node)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node */
	if (pe__is_guest_node(node)) {
	node_t *host_node = pe__current_node(node->details->remote_rsc);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_clone_detail && safe_str_neq(node->details->uname, node->details->id)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = malloc(name_len);
	CRM_ASSERT(node_name != NULL);
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	/*!
	* \internal
	* \brief Print a negative location constraint
	*
	* \param[in] stream File stream to display output to
	* \param[in] node Node affected by constraint
	* \param[in] location Constraint to print
	*/
	static void
	print_ban(FILE stream, pe_node_t node, pe__location_t *location)
	{
	char *node_name = NULL;

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	node_name = get_node_display_name(node);
	print_as(" %s\tprevents %s from running %son %s\n",
	location->id, location->rsc_lh->id,
	((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
	node_name);
	break;

	case mon_output_html:
	case mon_output_cgi:
	node_name = get_node_display_name(node);
	fprintf(stream, " <li>%s prevents %s from running %son %s</li>\n",
	location->id, location->rsc_lh->id,
	((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
	node_name);
	break;

	case mon_output_xml:
	fprintf(stream,
	" <ban id=\"%s\" resource=\"%s\" node=\"%s\" weight=\"%d\" master_only=\"%s\" />\n",
	location->id, location->rsc_lh->id, node->details->uname, node->weight,
	((location->role_filter == RSC_ROLE_MASTER)? "true" : "false"));
	break;

	default:
	break;
	}
	free(node_name);
	}

	/*!
	* \internal
	* \brief Print section for negative location constraints
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set corresponding to CIB status to display
	*/
	static void
	print_neg_locations(FILE stream, pe_working_set_t data_set)
	{
	GListPtr gIter, gIter2;

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nNegative Location Constraints:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Negative Location Constraints</h2>\n <ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <bans>\n");
	break;

	default:
	break;
	}

	/* Print each ban */
	for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
	pe__location_t *location = gIter->data;
	if (!g_str_has_prefix(location->id, print_neg_location_prefix))
	continue;
	for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
	node_t node = (node_t ) gIter2->data;

	if (node->weight < 0) {
	print_ban(stream, node, location);
	}
	}
	}

	/* Close section */
	switch (output_format) {
	case mon_output_cgi:
	case mon_output_html:
	fprintf(stream, " </ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </bans>\n");
	break;

	default:
	break;
	}
	}

	static void
	crm_mon_get_parameters(resource_t rsc, pe_working_set_t data_set)
	{
	get_rsc_attributes(rsc->parameters, rsc, NULL, data_set);
	crm_trace("Beekhof: unpacked params for %s (%d)", rsc->id, g_hash_table_size(rsc->parameters));
	if(rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	crm_mon_get_parameters(gIter->data, data_set);
	}
	}
	}

	/*!
	* \internal
	* \brief Print node attributes section
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_node_attributes(FILE stream, pe_working_set_t data_set)
	{
	GListPtr gIter = NULL;

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nNode Attributes:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Node Attributes</h2>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <node_attributes>\n");
	break;

	default:
	break;
	}

	/* Unpack all resource parameters (it would be more efficient to do this
	* only when needed for the first time in print_attr_msg())
	*/
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	crm_mon_get_parameters(gIter->data, data_set);
	}

	/* Display each node's attributes */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	struct mon_attr_data data;

	data.stream = stream;
	data.node = (node_t *) gIter->data;

	if (data.node && data.node->details && data.node->details->online) {
	print_node_start(stream, data.node);
	g_hash_table_foreach(data.node->details->attrs, create_attr_list, NULL);
	g_list_foreach(attr_list, print_node_attribute, &data);
	g_list_free(attr_list);
	attr_list = NULL;
	print_node_end(stream);
	}
	}

	/* Print section footer */
	switch (output_format) {
	case mon_output_xml:
	fprintf(stream, " </node_attributes>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Return resource display options corresponding to command-line choices
	*
	* \return Bitmask of pe_print_options suitable for resource print functions
	*/
	static int
	get_resource_display_options(void)
	{
	int print_opts;

	/* Determine basic output format */
	switch (output_format) {
	case mon_output_console:
	print_opts = pe_print_ncurses;
	break;
	case mon_output_html:
	case mon_output_cgi:
	print_opts = pe_print_html;
	break;
	case mon_output_xml:
	print_opts = pe_print_xml;
	break;
	default:
	print_opts = pe_print_printf;
	break;
	}

	/* Add optional display elements */
	if (print_pending) {
	print_opts \|= pe_print_pending;
	}
	if (print_clone_detail) {
	print_opts \|= pe_print_clone_details\|pe_print_implicit;
	}
	if (!inactive_resources) {
	print_opts \|= pe_print_clone_active;
	}
	if (print_brief) {
	print_opts \|= pe_print_brief;
	}
	return print_opts;
	}

	/*!
	* \internal
	* \brief Return human-friendly string representing current time
	*
	* \return Current time as string (as by ctime() but without newline) on success
	* or "Could not determine current time" on error
	* \note The return value points to a statically allocated string which might be
	* overwritten by subsequent calls to any of the C library date and time functions.
	*/
	static const char *
	crm_now_string(void)
	{
	time_t a_time = time(NULL);
	char *since_epoch = ctime(&a_time);

	if ((a_time == (time_t) -1) \|\| (since_epoch == NULL)) {
	return "Could not determine current time";
	}
	since_epoch[strlen(since_epoch) - 1] = EOS; /* trim newline */
	return (since_epoch);
	}

	/*!
	* \internal
	* \brief Print header for cluster summary if needed
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_cluster_summary_header(FILE *stream)
	{
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <h2>Cluster Summary</h2>\n <p>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <summary>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print footer for cluster summary if needed
	*
	* \param[in] stream File stream to display output to
	*/
	static void
	print_cluster_summary_footer(FILE *stream)
	{
	switch (output_format) {
	case mon_output_cgi:
	case mon_output_html:
	fprintf(stream, " </p>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </summary>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print times the display was last updated and CIB last changed
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_cluster_times(FILE stream, pe_working_set_t data_set)
	{
	const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
	const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
	const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
	const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("Last updated: %s", crm_now_string());
	print_as((user \|\| client \|\| origin)? "\n" : "\t\t");
	print_as("Last change: %s", last_written ? last_written : "");
	if (user) {
	print_as(" by %s", user);
	}
	if (client) {
	print_as(" via %s", client);
	}
	if (origin) {
	print_as(" on %s", origin);
	}
	print_as("\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <b>Last updated:</b> %s<br/>\n", crm_now_string());
	fprintf(stream, " <b>Last change:</b> %s", last_written ? last_written : "");
	if (user) {
	fprintf(stream, " by %s", user);
	}
	if (client) {
	fprintf(stream, " via %s", client);
	}
	if (origin) {
	fprintf(stream, " on %s", origin);
	}
	fprintf(stream, "<br/>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <last_update time=\"%s\" />\n", crm_now_string());
	fprintf(stream, " <last_change time=\"%s\" user=\"%s\" client=\"%s\" origin=\"%s\" />\n",
	last_written ? last_written : "", user ? user : "",
	client ? client : "", origin ? origin : "");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print cluster stack
	*
	* \param[in] stream File stream to display output to
	* \param[in] stack_s Stack name
	*/
	static void
	print_cluster_stack(FILE stream, const char stack_s)
	{
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("Stack: %s\n", stack_s);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <b>Stack:</b> %s<br/>\n", stack_s);
	break;

	case mon_output_xml:
	fprintf(stream, " <stack type=\"%s\" />\n", stack_s);
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print current DC and its version
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_cluster_dc(FILE stream, pe_working_set_t data_set)
	{
	node_t *dc = data_set->dc_node;
	xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
	data_set->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
	: NULL;
	const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
	char *dc_name = dc? get_node_display_name(dc) : NULL;

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("Current DC: ");
	if (dc) {
	print_as("%s (version %s) - partition %s quorum\n",
	dc_name, (dc_version_s? dc_version_s : "unknown"),
	(crm_is_true(quorum) ? "with" : "WITHOUT"));
	} else {
	print_as("NONE\n");
	}
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <b>Current DC:</b> ");
	if (dc) {
	fprintf(stream, "%s (version %s) - partition %s quorum",
	dc_name, (dc_version_s? dc_version_s : "unknown"),
	(crm_is_true(quorum)? "with" : "<font color=\"red\"><b>WITHOUT</b></font>"));
	} else {
	fprintf(stream, "<font color=\"red\"><b>NONE</b></font>");
	}
	fprintf(stream, "<br/>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <current_dc ");
	if (dc) {
	fprintf(stream,
	"present=\"true\" version=\"%s\" name=\"%s\" id=\"%s\" with_quorum=\"%s\"",
	(dc_version_s? dc_version_s : ""), dc->details->uname, dc->details->id,
	(crm_is_true(quorum) ? "true" : "false"));
	} else {
	fprintf(stream, "present=\"false\"");
	}
	fprintf(stream, " />\n");
	break;

	default:
	break;
	}
	free(dc_name);
	}

	/*!
	* \internal
	* \brief Print counts of configured nodes and resources
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	* \param[in] stack_s Stack name
	*/
	static void
	print_cluster_counts(FILE stream, pe_working_set_t data_set, const char *stack_s)
	{
	int nnodes = g_list_length(data_set->nodes);
	int nresources = count_resources(data_set, NULL);

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:

	print_as("\n%d node%s configured\n", nnodes, s_if_plural(nnodes));

	print_as("%d resource%s configured",
	nresources, s_if_plural(nresources));
	if(data_set->disabled_resources \|\| data_set->blocked_resources) {
	print_as(" (");
	if (data_set->disabled_resources) {
	print_as("%d DISABLED", data_set->disabled_resources);
	}
	if (data_set->disabled_resources && data_set->blocked_resources) {
	print_as(", ");
	}
	if (data_set->blocked_resources) {
	print_as("%d BLOCKED from starting due to failure",
	data_set->blocked_resources);
	}
	print_as(")");
	}
	print_as("\n");

	break;

	case mon_output_html:
	case mon_output_cgi:

	fprintf(stream, " %d node%s configured<br/>\n",
	nnodes, s_if_plural(nnodes));

	fprintf(stream, " %d resource%s configured",
	nresources, s_if_plural(nresources));
	if (data_set->disabled_resources \|\| data_set->blocked_resources) {
	fprintf(stream, " (");
	if (data_set->disabled_resources) {
	fprintf(stream, "%d <strong>DISABLED</strong>",
	data_set->disabled_resources);
	}
	if (data_set->disabled_resources && data_set->blocked_resources) {
	fprintf(stream, ", ");
	}
	if (data_set->blocked_resources) {
	fprintf(stream,
	"%d <strong>BLOCKED</strong> from starting due to failure",
	data_set->blocked_resources);
	}
	fprintf(stream, ")");
	}
	fprintf(stream, "<br/>\n");
	break;

	case mon_output_xml:
	fprintf(stream,
	" <nodes_configured number=\"%d\" />\n",
	g_list_length(data_set->nodes));
	fprintf(stream,
	" <resources_configured number=\"%d\" disabled=\"%d\" blocked=\"%d\" />\n",
	count_resources(data_set, NULL),
	data_set->disabled_resources, data_set->blocked_resources);
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print cluster-wide options
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*
	* \note Currently this is only implemented for HTML and XML output, and
	* prints only a few options. If there is demand, more could be added.
	*/
	static void
	print_cluster_options(FILE stream, pe_working_set_t data_set)
	{
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
	print_as("\n * Resource management is DISABLED *");
	print_as("\n The cluster will not attempt to start, stop or recover services");
	print_as("\n");
	}
	break;

	case mon_output_html:
	fprintf(stream, " </p>\n <h3>Config Options</h3>\n");
	fprintf(stream, " <table>\n");
	fprintf(stream, " <tr><th>STONITH of failed nodes</th><td>%s</td></tr>\n",
	is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");

	fprintf(stream, " <tr><th>Cluster is</th><td>%ssymmetric</td></tr>\n",
	is_set(data_set->flags, pe_flag_symmetric_cluster)? "" : "a");

	fprintf(stream, " <tr><th>No Quorum Policy</th><td>");
	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	fprintf(stream, "Freeze resources");
	break;
	case no_quorum_stop:
	fprintf(stream, "Stop ALL resources");
	break;
	case no_quorum_ignore:
	fprintf(stream, "Ignore");
	break;
	case no_quorum_suicide:
	fprintf(stream, "Suicide");
	break;
	}
	fprintf(stream, "</td></tr>\n");

	fprintf(stream, " <tr><th>Resource management</th><td>");
	if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
	fprintf(stream, "<strong>DISABLED</strong> (the cluster will "
	"not attempt to start, stop or recover services)");
	} else {
	fprintf(stream, "enabled");
	}
	fprintf(stream, "</td></tr>\n");

	fprintf(stream, "</table>\n <p>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <cluster_options");
	fprintf(stream, " stonith-enabled=\"%s\"",
	is_set(data_set->flags, pe_flag_stonith_enabled)?
	"true" : "false");
	fprintf(stream, " symmetric-cluster=\"%s\"",
	is_set(data_set->flags, pe_flag_symmetric_cluster)?
	"true" : "false");
	fprintf(stream, " no-quorum-policy=\"");
	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	fprintf(stream, "freeze");
	break;
	case no_quorum_stop:
	fprintf(stream, "stop");
	break;
	case no_quorum_ignore:
	fprintf(stream, "ignore");
	break;
	case no_quorum_suicide:
	fprintf(stream, "suicide");
	break;
	}
	fprintf(stream, "\"");
	fprintf(stream, " maintenance-mode=\"%s\"",
	is_set(data_set->flags, pe_flag_maintenance_mode)?
	"true" : "false");
	fprintf(stream, " />\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Get the name of the stack in use (or "unknown" if not available)
	*
	* \param[in] data_set Working set of CIB state
	*
	* \return String representing stack name
	*/
	static const char *
	get_cluster_stack(pe_working_set_t *data_set)
	{
	xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
	data_set->input, LOG_DEBUG);
	return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
	}

	/*!
	* \internal
	* \brief Print a summary of cluster-wide information
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_cluster_summary(FILE stream, pe_working_set_t data_set)
	{
	const char *stack_s = get_cluster_stack(data_set);
	gboolean header_printed = FALSE;

	if (show & mon_show_stack) {
	if (header_printed == FALSE) {
	print_cluster_summary_header(stream);
	header_printed = TRUE;
	}
	print_cluster_stack(stream, stack_s);
	}

	/* Always print DC if none, even if not requested */
	if ((data_set->dc_node == NULL) \|\| (show & mon_show_dc)) {
	if (header_printed == FALSE) {
	print_cluster_summary_header(stream);
	header_printed = TRUE;
	}
	print_cluster_dc(stream, data_set);
	}

	if (show & mon_show_times) {
	if (header_printed == FALSE) {
	print_cluster_summary_header(stream);
	header_printed = TRUE;
	}
	print_cluster_times(stream, data_set);
	}

	if (is_set(data_set->flags, pe_flag_maintenance_mode)
	\|\| data_set->disabled_resources
	\|\| data_set->blocked_resources
	\|\| is_set(show, mon_show_count)) {
	if (header_printed == FALSE) {
	print_cluster_summary_header(stream);
	header_printed = TRUE;
	}
	print_cluster_counts(stream, data_set, stack_s);
	}

	/* There is not a separate option for showing cluster options, so show with
	* stack for now; a separate option could be added if there is demand
	*/
	if (show & mon_show_stack) {
	print_cluster_options(stream, data_set);
	}

	if (header_printed) {
	print_cluster_summary_footer(stream);
	}
	}

	/*!
	* \internal
	* \brief Print a failed action
	*
	* \param[in] stream File stream to display output to
	* \param[in] xml_op Root of XML tree describing failed action
	*/
	static void
	print_failed_action(FILE stream, xmlNode xml_op)
	{
	const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	const char *op_key_attr = "op_key";
	const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
	const char *node = crm_element_value(xml_op, XML_ATTR_UNAME);
	const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
	int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
	int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
	char *exit_reason_cleaned;

	/* If no op_key was given, use id instead */
	if (op_key == NULL) {
	op_key = ID(xml_op);
	op_key_attr = "id";
	}

	/* If no exit reason was given, use "none" */
	if (exit_reason == NULL) {
	exit_reason = "none";
	}

	/* Print common action information */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("* %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
	op_key, node, services_ocf_exitcode_str(rc), rc,
	call, services_lrm_status_str(status), exit_reason);
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <li>%s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
	op_key, node, services_ocf_exitcode_str(rc), rc,
	call, services_lrm_status_str(status), exit_reason);
	break;

	case mon_output_xml:
	exit_reason_cleaned = crm_xml_escape(exit_reason);
	fprintf(stream, " <failure %s=\"%s\" node=\"%s\"",
	op_key_attr, op_key, node);
	fprintf(stream, " exitstatus=\"%s\" exitreason=\"%s\" exitcode=\"%d\"",
	services_ocf_exitcode_str(rc), exit_reason_cleaned, rc);
	fprintf(stream, " call=\"%s\" status=\"%s\"",
	call, services_lrm_status_str(status));
	free(exit_reason_cleaned);
	break;

	default:
	break;
	}

	/* If last change was given, print timing information as well */
	if (last) {
	time_t run_at = crm_parse_int(last, "0");
	char *run_at_s = ctime(&run_at);

	if (run_at_s) {
	run_at_s[24] = 0; /* Overwrite the newline */
	}

	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as(",\n last-rc-change='%s', queued=%sms, exec=%sms",
	run_at_s? run_at_s : "",
	crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " last-rc-change='%s', queued=%sms, exec=%sms",
	run_at_s? run_at_s : "",
	crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	break;

	case mon_output_xml:
	fprintf(stream,
	" last-rc-change=\"%s\" queued=\"%s\" exec=\"%s\" interval=\"%u\" task=\"%s\"",
	run_at_s? run_at_s : "",
	crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	crm_parse_ms(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS)),
	crm_element_value(xml_op, XML_LRM_ATTR_TASK));
	break;

	default:
	break;
	}
	}

	/* End the action listing */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, "</li>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " />\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print a section for failed actions
	*
	* \param[in] stream File stream to display output to
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_failed_actions(FILE stream, pe_working_set_t data_set)
	{
	xmlNode *xml_op = NULL;

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nFailed Resource Actions:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream,
	" <hr />\n <h2>Failed Resource Actions</h2>\n <ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <failures>\n");
	break;

	default:
	break;
	}

	/* Print each failed action */
	for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
	xml_op = __xml_next(xml_op)) {
	print_failed_action(stream, xml_op);
	}

	/* End section */
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </failures>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Reduce the stonith-history
	* for successful actions we keep the last of every action-type & target
	* for failed actions we record as well who had failed
	* for actions in progress we keep full track
	*
	* \param[in] history List of stonith actions
	*
	*/
	static stonith_history_t *
	reduce_stonith_history(stonith_history_t *history)
	{
	stonith_history_t new = NULL, hp, np, tmp;

	for (hp = history; hp; ) {
	for (np = new; np; np = np->next) {
	if ((hp->state == st_done) \|\| (hp->state == st_failed)) {
	/* action not in progress */
	if (safe_str_eq(hp->target, np->target) &&
	safe_str_eq(hp->action, np->action) &&
	(hp->state == np->state)) {
	if ((hp->state == st_done) \|\|
	safe_str_eq(hp->delegate, np->delegate)) {
	/* replace or purge */
	if (hp->completed < np->completed) {
	/* purge older hp */
	tmp = hp->next;
	hp->next = NULL;
	stonith_history_free(hp);
	hp = tmp;
	break;
	}
	/* damn single linked list */
	free(hp->target);
	free(hp->action);
	free(np->origin);
	np->origin = hp->origin;
	free(np->delegate);
	np->delegate = hp->delegate;
	free(np->client);
	np->client = hp->client;
	np->completed = hp->completed;
	tmp = hp;
	hp = hp->next;
	free(tmp);
	break;
	}
	}
	if (np->next) {
	continue;
	}
	}
	np = 0; /* let outer loop progress hp */
	break;
	}
	/* simply move hp from history to new */
	if (np == NULL) {
	tmp = hp->next;
	hp->next = new;
	new = hp;
	hp = tmp;
	}
	}
	return new;
	}

	/*!
	* \internal
	* \brief Sort the stonith-history
	* sort by competed most current on the top
	* pending actions lacking a completed-stamp are gathered at the top
	*
	* \param[in] history List of stonith actions
	*
	*/
	static stonith_history_t *
	sort_stonith_history(stonith_history_t *history)
	{
	stonith_history_t new = NULL, pending = NULL, hp, np, *tmp;

	for (hp = history; hp; ) {
	tmp = hp->next;
	if ((hp->state == st_done) \|\| (hp->state == st_failed)) {
	/* sort into new */
	if ((!new) \|\| (hp->completed > new->completed)) {
	hp->next = new;
	new = hp;
	} else {
	np = new;
	do {
	if ((!np->next) \|\| (hp->completed > np->next->completed)) {
	hp->next = np->next;
	np->next = hp;
	break;
	}
	np = np->next;
	} while (1);
	}
	} else {
	/* put into pending */
	hp->next = pending;
	pending = hp;
	}
	hp = tmp;
	}

	/* pending actions don't have a completed-stamp so make them go front */
	if (pending) {
	stonith_history_t *last_pending = pending;

	while (last_pending->next) {
	last_pending = last_pending->next;
	}

	last_pending->next = new;
	new = pending;
	}
	return new;
	}

	/*!
	* \internal
	* \brief Print a stonith action
	*
	* \param[in] stream File stream to display output to
	* \param[in] event stonith event
	*/
	static void
	print_stonith_action(FILE stream, stonith_history_t event)
	{
	const char *action_s = stonith_action_str(event->action);
	char *run_at_s = ctime(&event->completed);

	if ((run_at_s) && (run_at_s[0] != 0)) {
	run_at_s[strlen(run_at_s)-1] = 0; /* Overwrite the newline */
	}

	switch(output_format) {
	case mon_output_xml:
	fprintf(stream, " <fence_event target=\"%s\" action=\"%s\"",
	event->target, event->action);
	switch(event->state) {
	case st_done:
	fprintf(stream, " state=\"success\"");
	break;
	case st_failed:
	fprintf(stream, " state=\"failed\"");
	break;
	default:
	fprintf(stream, " state=\"pending\"");
	}
	fprintf(stream, " origin=\"%s\" client=\"%s\"",
	event->origin, event->client);
	if (event->delegate) {
	fprintf(stream, " delegate=\"%s\"", event->delegate);
	}
	switch(event->state) {
	case st_done:
	case st_failed:
	fprintf(stream, " completed=\"%s\"", run_at_s?run_at_s:"");
	break;
	default:
	break;
	}
	fprintf(stream, " />\n");
	break;

	case mon_output_plain:
	case mon_output_console:
	switch(event->state) {
	case st_done:
	print_as("* %s of %s successful: delegate=%s, client=%s, origin=%s,\n"
	" %s='%s'\n",
	action_s, event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	fence_full_history?"completed":"last-successful",
	run_at_s?run_at_s:"");
	break;
	case st_failed:
	print_as("* %s of %s failed: delegate=%s, client=%s, origin=%s,\n"
	" %s='%s'\n",
	action_s, event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	fence_full_history?"completed":"last-failed",
	run_at_s?run_at_s:"");
	break;
	default:
	print_as("* %s of %s pending: client=%s, origin=%s\n",
	action_s, event->target,
	event->client, event->origin);
	}
	break;

	case mon_output_html:
	case mon_output_cgi:
	switch(event->state) {
	case st_done:
	fprintf(stream, " <li>%s of %s successful: delegate=%s, "
	"client=%s, origin=%s, %s='%s'</li>\n",
	action_s, event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	fence_full_history?"completed":"last-successful",
	run_at_s?run_at_s:"");
	break;
	case st_failed:
	fprintf(stream, " <li>%s of %s failed: delegate=%s, "
	"client=%s, origin=%s, %s='%s'</li>\n",
	action_s, event->target,
	event->delegate ? event->delegate : "",
	event->client, event->origin,
	fence_full_history?"completed":"last-failed",
	run_at_s?run_at_s:"");
	break;
	default:
	fprintf(stream, " <li>%s of %s pending: client=%s, "
	"origin=%s</li>\n",
	action_s, event->target,
	event->client, event->origin);
	}
	break;

	default:
	/* no support for fence history for other formats so far */
	break;
	}
	}

	/*!
	* \internal
	* \brief Print a section for failed stonith actions
	*
	* \param[in] stream File stream to display output to
	* \param[in] history List of stonith actions
	*
	*/
	static void
	print_failed_stonith_actions(FILE stream, stonith_history_t history)
	{
	stonith_history_t *hp;

	for (hp = history; hp; hp = hp->next) {
	if (hp->state == st_failed) {
	break;
	}
	}
	if (!hp) {
	return;
	}

	/* Print section heading */
	switch (output_format) {
	/* no need to take care of xml in here as xml gets full
	* history anyway
	*/
	case mon_output_plain:
	case mon_output_console:
	print_as("\nFailed Fencing Actions:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Failed Fencing Actions</h2>\n <ul>\n");
	break;

	default:
	break;
	}

	/* Print each failed stonith action */
	for (hp = history; hp; hp = hp->next) {
	if (hp->state == st_failed) {
	print_stonith_action(stream, hp);
	}
	}

	/* End section */
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print pending stonith actions
	*
	* \param[in] stream File stream to display output to
	* \param[in] history List of stonith actions
	*
	*/
	static void
	print_stonith_pending(FILE stream, stonith_history_t history)
	{
	/* xml-output always shows the full history
	* so we'll never have to show pending-actions
	* separately
	*/
	if (history && (history->state != st_failed) &&
	(history->state != st_done)) {
	stonith_history_t *hp;

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nPending Fencing Actions:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Pending Fencing Actions</h2>\n <ul>\n");
	break;

	default:
	break;
	}

	for (hp = history; hp; hp = hp->next) {
	if ((hp->state == st_failed) \|\| (hp->state == st_done)) {
	break;
	}
	print_stonith_action(stream, hp);
	}

	/* End section */
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	default:
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Print a section for stonith-history
	*
	* \param[in] stream File stream to display output to
	* \param[in] history List of stonith actions
	*
	*/
	static void
	print_stonith_history(FILE stream, stonith_history_t history)
	{
	stonith_history_t *hp;

	/* Print section heading */
	switch (output_format) {
	case mon_output_plain:
	case mon_output_console:
	print_as("\nFencing History:\n");
	break;

	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " <hr />\n <h2>Fencing History</h2>\n <ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " <fence_history>\n");
	break;

	default:
	break;
	}

	for (hp = history; hp; hp = hp->next) {
	if ((hp->state != st_failed) \|\| (output_format == mon_output_xml)) {
	print_stonith_action(stream, hp);
	}
	}

	/* End section */
	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	fprintf(stream, " </ul>\n");
	break;

	case mon_output_xml:
	fprintf(stream, " </fence_history>\n");
	break;

	default:
	break;
	}
	}

	/*!
	* \internal
	* \brief Print cluster status to screen
	*
	* This uses the global display preferences set by command-line options
	* to display cluster status in a human-friendly way.
	*
	* \param[in] data_set Working set of CIB state
	* \param[in] stonith_history List of stonith actions
	*/
	static void
	print_status(pe_working_set_t * data_set,
	stonith_history_t *stonith_history)
	{
	GListPtr gIter = NULL;
	int print_opts = get_resource_display_options();

	/* space-separated lists of node names */
	char *online_nodes = NULL;
	char *online_remote_nodes = NULL;
	char *online_guest_nodes = NULL;
	char *offline_nodes = NULL;
	char *offline_remote_nodes = NULL;

	if (output_format == mon_output_console) {
	blank_screen();
	}
	print_cluster_summary(stdout, data_set);
	print_as("\n");

	/* Gather node information (and print if in bad state or grouping by node) */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	const char *node_mode = NULL;
	char *node_name = get_node_display_name(node);

	/* Get node mode */
	if (node->details->unclean) {
	if (node->details->online) {
	node_mode = "UNCLEAN (online)";

	} else if (node->details->pending) {
	node_mode = "UNCLEAN (pending)";

	} else {
	node_mode = "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	node_mode = "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	node_mode = "standby (on-fail)";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	node_mode = "standby (with active resources)";
	} else {
	node_mode = "standby";
	}
	} else {
	node_mode = "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	node_mode = "maintenance";
	} else {
	node_mode = "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	node_mode = "online";
	if (group_by_node == FALSE) {
	if (pe__is_guest_node(node)) {
	online_guest_nodes = add_list_element(online_guest_nodes, node_name);
	} else if (pe__is_remote_node(node)) {
	online_remote_nodes = add_list_element(online_remote_nodes, node_name);
	} else {
	online_nodes = add_list_element(online_nodes, node_name);
	}
	free(node_name);
	continue;
	}
	} else {
	node_mode = "OFFLINE";
	if (group_by_node == FALSE) {
	if (pe__is_remote_node(node)) {
	offline_remote_nodes = add_list_element(offline_remote_nodes, node_name);
	} else if (pe__is_guest_node(node)) {
	/* ignore offline guest nodes */
	} else {
	offline_nodes = add_list_element(offline_nodes, node_name);
	}
	free(node_name);
	continue;
	}
	}

	/* If we get here, node is in bad state, or we're grouping by node */

	/* Print the node name and status */
	if (pe__is_guest_node(node)) {
	print_as("Guest");
	} else if (pe__is_remote_node(node)) {
	print_as("Remote");
	}
	print_as("Node %s: %s\n", node_name, node_mode);

	/* If we're grouping by node, print its resources */
	if (group_by_node) {
	if (print_brief) {
	print_rscs_brief(node->details->running_rsc, "\t", print_opts \| pe_print_rsconly,
	stdout, FALSE);
	} else {
	GListPtr gIter2 = NULL;

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	resource_t rsc = (resource_t ) gIter2->data;

	rsc->fns->print(rsc, "\t", print_opts \| pe_print_rsconly, stdout);
	}
	}
	}
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes) {
	print_as("Online: [%s ]\n", online_nodes);
	free(online_nodes);
	}
	if (offline_nodes) {
	print_as("OFFLINE: [%s ]\n", offline_nodes);
	free(offline_nodes);
	}
	if (online_remote_nodes) {
	print_as("RemoteOnline: [%s ]\n", online_remote_nodes);
	free(online_remote_nodes);
	}
	if (offline_remote_nodes) {
	print_as("RemoteOFFLINE: [%s ]\n", offline_remote_nodes);
	free(offline_remote_nodes);
	}
	if (online_guest_nodes) {
	print_as("GuestOnline: [%s ]\n", online_guest_nodes);
	free(online_guest_nodes);
	}

	/* Print resources section, if needed */
	print_resources(stdout, data_set, print_opts);

	/* print Node Attributes section if requested */
	if (show & mon_show_attributes) {
	print_node_attributes(stdout, data_set);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (show & (mon_show_operations \| mon_show_failcounts)) {
	print_node_summary(stdout, data_set,
	((show & mon_show_operations)? TRUE : FALSE));
	}

	/* If there were any failed actions, print them */
	if (xml_has_children(data_set->failed)) {
	print_failed_actions(stdout, data_set);
	}

	/* Print failed stonith actions */
	if (fence_history) {
	print_failed_stonith_actions(stdout, stonith_history);
	}

	/* Print tickets if requested */
	if (show & mon_show_tickets) {
	print_cluster_tickets(stdout, data_set);
	}

	/* Print negative location constraints if requested */
	if (show & mon_show_bans) {
	print_neg_locations(stdout, data_set);
	}

	/* Print stonith history */
	if (fence_history) {
	if (show & mon_show_fence_history) {
	print_stonith_history(stdout, stonith_history);
	} else {
	print_stonith_pending(stdout, stonith_history);
	}
	}

	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	refresh();
	}
	#endif
	}

	/*!
	* \internal
	* \brief Print cluster status in XML format
	*
	* \param[in] data_set Working set of CIB state
	*/
	static void
	print_xml_status(pe_working_set_t * data_set,
	stonith_history_t *stonith_history)
	{
	FILE *stream = stdout;
	GListPtr gIter = NULL;
	int print_opts = get_resource_display_options();

	fprintf(stream, "<?xml version=\"1.0\"?>\n");
	fprintf(stream, "<crm_mon version=\"%s\">\n", VERSION);

	print_cluster_summary(stream, data_set);

	/* NODES */
	fprintf(stream, " <nodes>\n");
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	const char *node_type = "unknown";

	switch (node->details->type) {
	case node_member:
	node_type = "member";
	break;
	case node_remote:
	node_type = "remote";
	break;
	case node_ping:
	node_type = "ping";
	break;
	}

	fprintf(stream, " <node name=\"%s\" ", node->details->uname);
	fprintf(stream, "id=\"%s\" ", node->details->id);
	fprintf(stream, "online=\"%s\" ", node->details->online ? "true" : "false");
	fprintf(stream, "standby=\"%s\" ", node->details->standby ? "true" : "false");
	fprintf(stream, "standby_onfail=\"%s\" ", node->details->standby_onfail ? "true" : "false");
	fprintf(stream, "maintenance=\"%s\" ", node->details->maintenance ? "true" : "false");
	fprintf(stream, "pending=\"%s\" ", node->details->pending ? "true" : "false");
	fprintf(stream, "unclean=\"%s\" ", node->details->unclean ? "true" : "false");
	fprintf(stream, "shutdown=\"%s\" ", node->details->shutdown ? "true" : "false");
	fprintf(stream, "expected_up=\"%s\" ", node->details->expected_up ? "true" : "false");
	fprintf(stream, "is_dc=\"%s\" ", node->details->is_dc ? "true" : "false");
	fprintf(stream, "resources_running=\"%d\" ", g_list_length(node->details->running_rsc));
	fprintf(stream, "type=\"%s\" ", node_type);
	if (pe__is_guest_node(node)) {
	fprintf(stream, "id_as_resource=\"%s\" ", node->details->remote_rsc->container->id);
	}

	if (group_by_node) {
	GListPtr lpc2 = NULL;

	fprintf(stream, ">\n");
	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	resource_t rsc = (resource_t ) lpc2->data;

	rsc->fns->print(rsc, " ", print_opts \| pe_print_rsconly, stream);
	}
	fprintf(stream, " </node>\n");
	} else {
	fprintf(stream, "/>\n");
	}
	}
	fprintf(stream, " </nodes>\n");

	/* Print resources section, if needed */
	print_resources(stream, data_set, print_opts);

	/* print Node Attributes section if requested */
	if (show & mon_show_attributes) {
	print_node_attributes(stream, data_set);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (show & (mon_show_operations \| mon_show_failcounts)) {
	print_node_summary(stream, data_set,
	((show & mon_show_operations)? TRUE : FALSE));
	}

	/* If there were any failed actions, print them */
	if (xml_has_children(data_set->failed)) {
	print_failed_actions(stream, data_set);
	}

	/* Print stonith history */
	if (fence_history) {
	print_stonith_history(stdout, stonith_history);
	}

	/* Print tickets if requested */
	if (show & mon_show_tickets) {
	print_cluster_tickets(stream, data_set);
	}

	/* Print negative location constraints if requested */
	if (show & mon_show_bans) {
	print_neg_locations(stream, data_set);
	}

	fprintf(stream, "</crm_mon>\n");
	fflush(stream);
	fclose(stream);
	}

	/*!
	* \internal
	* \brief Print cluster status in HTML format (with HTTP headers if CGI)
	*
	* \param[in] data_set Working set of CIB state
	* \param[in] filename Name of file to write HTML to (ignored if CGI)
	*
	* \return 0 on success, -1 on error
	*/
	static int
	print_html_status(pe_working_set_t * data_set,
	const char *filename,
	stonith_history_t *stonith_history)
	{
	FILE *stream;
	GListPtr gIter = NULL;
	char *filename_tmp = NULL;
	int print_opts = get_resource_display_options();

	if (output_format == mon_output_cgi) {
	stream = stdout;
	fprintf(stream, "Content-Type: text/html\n\n");

	} else {
	filename_tmp = crm_concat(filename, "tmp", '.');
	stream = fopen(filename_tmp, "w");
	if (stream == NULL) {
	crm_perror(LOG_ERR, "Cannot open %s for writing", filename_tmp);
	free(filename_tmp);
	return -1;
	}
	}

	fprintf(stream, "<html>\n");
	fprintf(stream, " <head>\n");
	fprintf(stream, " <title>Cluster status</title>\n");
	fprintf(stream, " <meta http-equiv=\"refresh\" content=\"%d\">\n", reconnect_msec / 1000);
	fprintf(stream, " </head>\n");
	fprintf(stream, "<body>\n");

	print_cluster_summary(stream, data_set);

	/* NODE LIST */

	fprintf(stream, " <hr />\n <h2>Node List</h2>\n");
	fprintf(stream, "<ul>\n");
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	char *node_name = get_node_display_name(node);

	fprintf(stream, "<li>Node: %s: ", node_name);
	if (node->details->standby_onfail && node->details->online) {
	fprintf(stream, "<font color=\"orange\">standby (on-fail)</font>\n");
	} else if (node->details->standby && node->details->online) {

	fprintf(stream, "<font color=\"orange\">standby%s</font>\n",
	node->details->running_rsc?" (with active resources)":"");
	} else if (node->details->standby) {
	fprintf(stream, "<font color=\"red\">OFFLINE (standby)</font>\n");
	} else if (node->details->maintenance && node->details->online) {
	fprintf(stream, "<font color=\"blue\">maintenance</font>\n");
	} else if (node->details->maintenance) {
	fprintf(stream, "<font color=\"red\">OFFLINE (maintenance)</font>\n");
	} else if (node->details->online) {
	fprintf(stream, "<font color=\"green\">online</font>\n");
	} else {
	fprintf(stream, "<font color=\"red\">OFFLINE</font>\n");
	}
	if (print_brief && group_by_node) {
	fprintf(stream, "<ul>\n");
	print_rscs_brief(node->details->running_rsc, NULL, print_opts \| pe_print_rsconly,
	stream, FALSE);
	fprintf(stream, "</ul>\n");

	} else if (group_by_node) {
	GListPtr lpc2 = NULL;

	fprintf(stream, "<ul>\n");
	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	resource_t rsc = (resource_t ) lpc2->data;

	fprintf(stream, "<li>");
	rsc->fns->print(rsc, NULL, print_opts \| pe_print_rsconly, stream);
	fprintf(stream, "</li>\n");
	}
	fprintf(stream, "</ul>\n");
	}
	fprintf(stream, "</li>\n");
	free(node_name);
	}
	fprintf(stream, "</ul>\n");

	/* Print resources section, if needed */
	print_resources(stream, data_set, print_opts);

	/* print Node Attributes section if requested */
	if (show & mon_show_attributes) {
	print_node_attributes(stream, data_set);
	}

	/* If requested, print resource operations (which includes failcounts)
	* or just failcounts
	*/
	if (show & (mon_show_operations \| mon_show_failcounts)) {
	print_node_summary(stream, data_set,
	((show & mon_show_operations)? TRUE : FALSE));
	}

	/* If there were any failed actions, print them */
	if (xml_has_children(data_set->failed)) {
	print_failed_actions(stream, data_set);
	}

	/* Print failed stonith actions */
	if (fence_history) {
	print_failed_stonith_actions(stream, stonith_history);
	}

	/* Print stonith history */
	if (fence_history) {
	if (show & mon_show_fence_history) {
	print_stonith_history(stream, stonith_history);
	} else {
	print_stonith_pending(stdout, stonith_history);
	}
	}

	/* Print tickets if requested */
	if (show & mon_show_tickets) {
	print_cluster_tickets(stream, data_set);
	}

	/* Print negative location constraints if requested */
	if (show & mon_show_bans) {
	print_neg_locations(stream, data_set);
	}

	fprintf(stream, "</body>\n");
	fprintf(stream, "</html>\n");
	fflush(stream);
	fclose(stream);

	if (output_format != mon_output_cgi) {
	if (rename(filename_tmp, filename) != 0) {
	crm_perror(LOG_ERR, "Unable to rename %s->%s", filename_tmp, filename);
	}
	free(filename_tmp);
	}
	return 0;
	}

	static int
	send_custom_trap(const char node, const char rsc, const char *task, int target_rc, int rc,
	int status, const char *desc)
	{
	pid_t pid;

	/setenv needs chars, these are ints /
	char *rc_s = crm_itoa(rc);
	char *status_s = crm_itoa(status);
	char *target_rc_s = crm_itoa(target_rc);

	crm_debug("Sending external notification to '%s' via '%s'", external_recipient, external_agent);

	if(rsc) {
	setenv("CRM_notify_rsc", rsc, 1);
	}
	if (external_recipient) {
	setenv("CRM_notify_recipient", external_recipient, 1);
	}
	setenv("CRM_notify_node", node, 1);
	setenv("CRM_notify_task", task, 1);
	setenv("CRM_notify_desc", desc, 1);
	setenv("CRM_notify_rc", rc_s, 1);
	setenv("CRM_notify_target_rc", target_rc_s, 1);
	setenv("CRM_notify_status", status_s, 1);

	pid = fork();
	if (pid == -1) {
	crm_perror(LOG_ERR, "notification fork() failed.");
	}
	if (pid == 0) {
	/* crm_debug("notification: I am the child. Executing the nofitication program."); */
	execl(external_agent, external_agent, NULL);
	exit(CRM_EX_ERROR);
	}

	crm_trace("Finished running custom notification program '%s'.", external_agent);
	free(target_rc_s);
	free(status_s);
	free(rc_s);
	return 0;
	}

	static void
	handle_rsc_op(xmlNode * xml, const char *node_id)
	{
	int rc = -1;
	int status = -1;
	int target_rc = -1;
	gboolean notify = TRUE;

	char *rsc = NULL;
	char *task = NULL;
	const char *desc = NULL;
	const char *magic = NULL;
	const char *id = NULL;
	const char *node = NULL;

	xmlNode *n = xml;
	xmlNode * rsc_op = xml;

	if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
	xmlNode *cIter;

	for(cIter = xml->children; cIter; cIter = cIter->next) {
	handle_rsc_op(cIter, node_id);
	}

	return;
	}

	id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
	if (id == NULL) {
	/* Compatibility with <= 1.1.5 */
	id = ID(rsc_op);
	}

	magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
	if (magic == NULL) {
	/* non-change */
	return;
	}

	if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
	&target_rc)) {
	crm_err("Invalid event %s detected for %s", magic, id);
	return;
	}

	if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
	crm_err("Invalid event detected for %s", id);
	goto bail;
	}

	node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);

	while (n != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(n))) {
	n = n->parent;
	}

	if(node == NULL && n) {
	node = crm_element_value(n, XML_ATTR_UNAME);
	}

	if (node == NULL && n) {
	node = ID(n);
	}

	if (node == NULL) {
	node = node_id;
	}

	if (node == NULL) {
	crm_err("No node detected for event %s (%s)", magic, id);
	goto bail;
	}

	/* look up where we expected it to be? */
	desc = pcmk_strerror(pcmk_ok);
	if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
	crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
	if (rc == PCMK_OCF_NOT_RUNNING) {
	notify = FALSE;
	}

	} else if (status == PCMK_LRM_OP_DONE) {
	desc = services_ocf_exitcode_str(rc);
	crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);

	} else {
	desc = services_lrm_status_str(status);
	crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
	}

	if (notify && external_agent) {
	send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
	}
	bail:
	free(rsc);
	free(task);
	}

	static gboolean
	mon_trigger_refresh(gpointer user_data)
	{
	mainloop_set_trigger(refresh_trigger);
	return FALSE;
	}

	#define NODE_PATT "/lrm[@id="
	static char *
	get_node_from_xpath(const char *xpath)
	{
	char *nodeid = NULL;
	char *tmp = strstr(xpath, NODE_PATT);

	if(tmp) {
	tmp += strlen(NODE_PATT);
	tmp += 1;

	nodeid = strdup(tmp);
	tmp = strstr(nodeid, "\'");
	CRM_ASSERT(tmp);
	tmp[0] = 0;
	}
	return nodeid;
	}

	static void
	crm_diff_update_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = __xml_first_child(diff); change != NULL; change = __xml_next(change)) {
	const char *name = NULL;
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);
	xmlNode *match = NULL;
	const char *node = NULL;

	if(op == NULL) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	match = change->children;

	} else if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "delete") == 0) {
	continue;

	} else if(strcmp(op, "modify") == 0) {
	match = first_named_child(change, XML_DIFF_RESULT);
	if(match) {
	match = match->children;
	}
	}

	if(match) {
	name = (const char *)match->name;
	}

	crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
	if(xpath == NULL) {
	/* Version field, ignore */

	} else if(name == NULL) {
	crm_debug("No result for %s operation to %s", op, xpath);
	CRM_ASSERT(strcmp(op, "delete") == 0 \|\| strcmp(op, "move") == 0);

	} else if(strcmp(name, XML_TAG_CIB) == 0) {
	xmlNode *state = NULL;
	xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);

	for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
	node = crm_element_value(state, XML_ATTR_UNAME);
	if (node == NULL) {
	node = ID(state);
	}
	handle_rsc_op(state, node);
	}

	} else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
	xmlNode *state = NULL;

	for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) {
	node = crm_element_value(state, XML_ATTR_UNAME);
	if (node == NULL) {
	node = ID(state);
	}
	handle_rsc_op(state, node);
	}

	} else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
	node = crm_element_value(match, XML_ATTR_UNAME);
	if (node == NULL) {
	node = ID(match);
	}
	handle_rsc_op(match, node);

	} else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
	node = ID(match);
	handle_rsc_op(match, node);

	} else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
	char *local_node = get_node_from_xpath(xpath);

	handle_rsc_op(match, local_node);
	free(local_node);

	} else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
	char *local_node = get_node_from_xpath(xpath);

	handle_rsc_op(match, local_node);
	free(local_node);

	} else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
	char *local_node = get_node_from_xpath(xpath);

	handle_rsc_op(match, local_node);
	free(local_node);

	} else {
	crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
	}
	}
	}

	static void
	crm_diff_update_v1(const char event, xmlNode msg)
	{
	/* Process operation updates */
	xmlXPathObject *xpathObj = xpath_search(msg,
	"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
	"//" XML_LRM_TAG_RSC_OP);
	int lpc = 0, max = numXpathResults(xpathObj);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);

	handle_rsc_op(rsc_op, NULL);
	}
	freeXpathObject(xpathObj);
	}

	static void
	crm_diff_update(const char event, xmlNode msg)
	{
	int rc = -1;
	static bool stale = FALSE;
	gboolean cib_updated = FALSE;
	xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	print_dot();

	if (current_cib != NULL) {
	rc = xml_apply_patchset(current_cib, diff, TRUE);

	switch (rc) {
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(current_cib); current_cib = NULL;
	break;
	case pcmk_ok:
	cib_updated = TRUE;
	break;
	default:
	crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(current_cib); current_cib = NULL;
	}
	}

	if (current_cib == NULL) {
	crm_trace("Re-requesting the full cib");
	cib->cmds->query(cib, NULL, &current_cib, cib_scope_local \| cib_sync_call);
	}

	if (external_agent) {
	int format = 0;
	crm_element_value_int(diff, "format", &format);
	switch(format) {
	case 1:
	crm_diff_update_v1(event, msg);
	break;
	case 2:
	crm_diff_update_v2(event, msg);
	break;
	default:
	crm_err("Unknown patch format: %d", format);
	}
	}

	if (current_cib == NULL) {
	if(!stale) {
	print_as("--- Stale data ---");
	}
	stale = TRUE;
	return;
	}

	stale = FALSE;
	kick_refresh(cib_updated);
	}

	static gboolean
	mon_refresh_display(gpointer user_data)
	{
	xmlNode *cib_copy = copy_xml(current_cib);
	stonith_history_t *stonith_history = NULL;

	last_refresh = time(NULL);

	if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
	if (cib) {
	cib->cmds->signoff(cib);
	}
	print_as("Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
	if (output_format == mon_output_console) {
	sleep(2);
	}
	clean_up(CRM_EX_CONFIG);
	return FALSE;
	}

	/* get the stonith-history if there is evidence we need it
	*/
	while (fence_history) {
	if (st != NULL) {
	if (st->cmds->history(st, st_opt_sync_call, NULL, &stonith_history, 120)) {
	fprintf(stderr, "Critical: Unable to get stonith-history\n");
	mon_cib_connection_destroy(NULL);
	} else {
	if ((!fence_full_history) && (output_format != mon_output_xml)) {
	stonith_history = reduce_stonith_history(stonith_history);
	}
	stonith_history = sort_stonith_history(stonith_history);
	break; /* all other cases are errors */
	}
	} else {
	fprintf(stderr, "Critical: No stonith-API\n");
	}
	free_xml(cib_copy);
	print_as("Reading stonith-history failed");
	if (output_format == mon_output_console) {
	sleep(2);
	}
	return FALSE;
	}

	if (mon_data_set == NULL) {
	mon_data_set = pe_new_working_set();
	CRM_ASSERT(mon_data_set != NULL);
	}

	mon_data_set->input = cib_copy;
	cluster_status(mon_data_set);

	/* Unpack constraints if any section will need them
	* (tickets may be referenced in constraints but not granted yet,
	* and bans need negative location constraints) */
	if (show & (mon_show_bans \| mon_show_tickets)) {
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
	mon_data_set->input);
	unpack_constraints(cib_constraints, mon_data_set);
	}

	switch (output_format) {
	case mon_output_html:
	case mon_output_cgi:
	if (print_html_status(mon_data_set, output_filename, stonith_history) != 0) {
	fprintf(stderr, "Critical: Unable to output html file\n");
	clean_up(CRM_EX_CANTCREAT);
	return FALSE;
	}
	break;

	case mon_output_xml:
	print_xml_status(mon_data_set, stonith_history);
	break;

	case mon_output_monitor:
	print_simple_status(mon_data_set, stonith_history);
	if (has_warnings) {
	clean_up(MON_STATUS_WARN);
	return FALSE;
	}
	break;

	case mon_output_plain:
	case mon_output_console:
	print_status(mon_data_set, stonith_history);
	break;

	case mon_output_none:
	break;
	}

	stonith_history_free(stonith_history);
	stonith_history = NULL;
	pe_reset_working_set(mon_data_set);
	return TRUE;
	}

	static void
	mon_st_callback_event(stonith_t * st, stonith_event_t * e)
	{
	if (st->state == stonith_disconnected) {
	/* disconnect cib as well and have everything reconnect */
	mon_cib_connection_destroy(NULL);
	} else if (external_agent) {
	char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
	e->operation, e->origin, e->target, pcmk_strerror(e->result),
	e->id);
	send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
	free(desc);
	}
	}

	static void
	kick_refresh(gboolean data_updated)
	{
	static int updates = 0;
	long now = time(NULL);

	if (data_updated) {
	updates++;
	}

	if(refresh_timer == NULL) {
	refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
	}

	/* Refresh
	* - immediately if the last update was more than 5s ago
	* - every 10 cib-updates
	* - at most 2s after the last update
	*/
	if ((now - last_refresh) > (reconnect_msec / 1000)) {
	mainloop_set_trigger(refresh_trigger);
	mainloop_timer_stop(refresh_timer);
	updates = 0;

	} else if(updates >= 10) {
	mainloop_set_trigger(refresh_trigger);
	mainloop_timer_stop(refresh_timer);
	updates = 0;

	} else {
	mainloop_timer_start(refresh_timer);
	}
	}

	static void
	mon_st_callback_display(stonith_t * st, stonith_event_t * e)
	{
	if (st->state == stonith_disconnected) {
	/* disconnect cib as well and have everything reconnect */
	mon_cib_connection_destroy(NULL);
	} else {
	print_dot();
	kick_refresh(TRUE);
	}
	}

	static void
	clean_up_connections(void)
	{
	if (cib != NULL) {
	cib->cmds->signoff(cib);
	cib_delete(cib);
	cib = NULL;
	}

	if (st != NULL) {
	if (st->state != stonith_disconnected) {
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
	st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY);
	st->cmds->disconnect(st);
	}
	stonith_api_delete(st);
	st = NULL;
	}
	}

	/*
	* De-init ncurses, disconnect from the CIB manager, disconnect fencing,
	* deallocate memory and show usage-message if requested.
	*
	* We don't actually return, but nominally returning crm_exit_t allows a usage
	* like "return clean_up(exit_code);" which helps static analysis understand the
	* code flow.
	*/
	static crm_exit_t
	clean_up(crm_exit_t exit_code)
	{
	#if CURSES_ENABLED
	if (curses_console_initialized) {
	output_format = mon_output_plain;
	echo();
	nocbreak();
	endwin();
	curses_console_initialized = FALSE;
	}
	#endif

	clean_up_connections();
	free(output_filename);
	free(pid_file);

	pe_free_working_set(mon_data_set);
	mon_data_set = NULL;

	if (exit_code == CRM_EX_USAGE) {
	if (output_format == mon_output_cgi) {
	fprintf(stdout, "Content-Type: text/plain\n"
	"Status: 500\n\n");
	} else {
	crm_help('?', CRM_EX_USAGE);
	}
	}
	crm_exit(exit_code);
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:08 PM (21 h, 30 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002494
Default Alt Text: (217 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions