No OneTemporary
Actions

Size

129 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/based/based_io.c b/daemons/based/based_io.c
	index 7815b38074..6d069ac91d 100644
	--- a/daemons/based/based_io.c
	+++ b/daemons/based/based_io.c
	@@ -1,467 +1,472 @@
	/*
	* Copyright 2004-2020 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <dirent.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>

	#include <crm/crm.h>

	#include <crm/cib.h>
	#include <crm/common/util.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>
	#include <crm/cluster.h>

	#include <pacemaker-based.h>

	crm_trigger_t *cib_writer = NULL;

	int write_cib_contents(gpointer p);

	static void
	cib_rename(const char *old)
	{
	int new_fd;
	char *new = crm_strdup_printf("%s/cib.auto.XXXXXX", cib_root);

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);
	new_fd = mkstemp(new);
	crm_err("Archiving unusable file %s as %s", old, new);
	if ((new_fd < 0) \|\| (rename(old, new) < 0)) {
	crm_perror(LOG_ERR, "Couldn't rename %s as %s", old, new);
	crm_err("Disabling disk writes and continuing");
	cib_writes_enabled = FALSE;
	}
	if (new_fd > 0) {
	close(new_fd);
	}
	free(new);
	}

	/*
	* It is the callers responsibility to free the output of this function
	*/

	static xmlNode *
	retrieveCib(const char filename, const char sigfile)
	{
	xmlNode *root = NULL;

	crm_info("Reading cluster configuration file %s (digest: %s)",
	filename, sigfile);
	switch (cib_file_read_and_verify(filename, sigfile, &root)) {
	case -pcmk_err_cib_corrupt:
	crm_warn("Continuing but %s will NOT be used.", filename);
	break;

	case -pcmk_err_cib_modified:
	/* Archive the original files so the contents are not lost */
	crm_warn("Continuing but %s will NOT be used.", filename);
	cib_rename(filename);
	cib_rename(sigfile);
	break;
	}
	return root;
	}

	/*
	* for OSs without support for direntry->d_type, like Solaris
	*/
	#ifndef DT_UNKNOWN
	# define DT_UNKNOWN 0
	# define DT_FIFO 1
	# define DT_CHR 2
	# define DT_DIR 4
	# define DT_BLK 6
	# define DT_REG 8
	# define DT_LNK 10
	# define DT_SOCK 12
	# define DT_WHT 14
	#endif /DT_UNKNOWN/

	static int cib_archive_filter(const struct dirent * a)
	{
	int rc = 0;
	/* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
	struct stat s;
	char *a_path = crm_strdup_printf("%s/%s", cib_root, a->d_name);

	if(stat(a_path, &s) != 0) {
	rc = errno;
	crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_strerror(rc), rc);
	rc = 0;

	} else if ((s.st_mode & S_IFREG) != S_IFREG) {
	unsigned char dtype;
	#ifdef HAVE_STRUCT_DIRENT_D_TYPE
	dtype = a->d_type;
	#else
	switch (s.st_mode & S_IFMT) {
	case S_IFREG: dtype = DT_REG; break;
	case S_IFDIR: dtype = DT_DIR; break;
	case S_IFCHR: dtype = DT_CHR; break;
	case S_IFBLK: dtype = DT_BLK; break;
	case S_IFLNK: dtype = DT_LNK; break;
	case S_IFIFO: dtype = DT_FIFO; break;
	case S_IFSOCK: dtype = DT_SOCK; break;
	default: dtype = DT_UNKNOWN; break;
	}
	#endif
	crm_trace("%s - wrong type (%d)", a->d_name, dtype);

	} else if(strstr(a->d_name, "cib-") != a->d_name) {
	crm_trace("%s - wrong prefix", a->d_name);

	} else if (pcmk__ends_with_ext(a->d_name, ".sig")) {
	crm_trace("%s - wrong suffix", a->d_name);

	} else {
	crm_debug("%s - candidate", a->d_name);
	rc = 1;
	}

	free(a_path);
	return rc;
	}

	static int cib_archive_sort(const struct dirent a, const struct dirent b)
	{
	/* Order by creation date - most recently created file first */
	int rc = 0;
	struct stat buf;

	time_t a_age = 0;
	time_t b_age = 0;

	char *a_path = crm_strdup_printf("%s/%s", cib_root, a[0]->d_name);
	char *b_path = crm_strdup_printf("%s/%s", cib_root, b[0]->d_name);

	if(stat(a_path, &buf) == 0) {
	a_age = buf.st_ctime;
	}
	if(stat(b_path, &buf) == 0) {
	b_age = buf.st_ctime;
	}

	free(a_path);
	free(b_path);

	if(a_age > b_age) {
	rc = 1;
	} else if(a_age < b_age) {
	rc = -1;
	}

	crm_trace("%s (%lu) vs. %s (%lu) : %d",
	a[0]->d_name, (unsigned long)a_age,
	b[0]->d_name, (unsigned long)b_age, rc);
	return rc;
	}

	xmlNode *
	readCibXmlFile(const char dir, const char file, gboolean discard_status)
	{
	struct dirent **namelist = NULL;

	int lpc = 0;
	char *sigfile = NULL;
	char *sigfilepath = NULL;
	char *filename = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *validation = NULL;
	const char *use_valgrind = getenv("PCMK_valgrind_enabled");

	xmlNode *root = NULL;
	xmlNode *status = NULL;

	sigfile = crm_strdup_printf("%s.sig", file);
	if (pcmk__daemon_can_write(dir, file) == FALSE
	\|\| pcmk__daemon_can_write(dir, sigfile) == FALSE) {
	cib_status = -EACCES;
	return NULL;
	}

	filename = crm_strdup_printf("%s/%s", dir, file);
	sigfilepath = crm_strdup_printf("%s/%s", dir, sigfile);
	free(sigfile);

	cib_status = pcmk_ok;
	root = retrieveCib(filename, sigfilepath);
	free(filename);
	free(sigfilepath);

	if (root == NULL) {
	crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
	lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
	if (lpc < 0) {
	crm_perror(LOG_NOTICE, "scandir(%s) failed", cib_root);
	}
	}

	while (root == NULL && lpc > 1) {
	crm_debug("Testing %d candidates", lpc);

	lpc--;

	filename = crm_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
	sigfile = crm_strdup_printf("%s.sig", filename);

	crm_info("Reading cluster configuration file %s (digest: %s)",
	filename, sigfile);
	if (cib_file_read_and_verify(filename, sigfile, &root) < 0) {
	crm_warn("Continuing but %s will NOT be used.", filename);
	} else {
	crm_notice("Continuing with last valid configuration archive: %s", filename);
	}

	free(namelist[lpc]);
	free(filename);
	free(sigfile);
	}
	free(namelist);

	if (root == NULL) {
	root = createEmptyCib(0);
	crm_warn("Continuing with an empty configuration.");
	}

	if (cib_writes_enabled && use_valgrind &&
	(crm_is_true(use_valgrind) \|\| strstr(use_valgrind, "pacemaker-based"))) {

	cib_writes_enabled = FALSE;
	crm_err("* Disabling disk writes to avoid confusing Valgrind *");
	}

	status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
	if (discard_status && status != NULL) {
	/* strip out the status section if there is one */
	free_xml(status);
	status = NULL;
	}
	if (status == NULL) {
	create_xml_node(root, XML_CIB_TAG_STATUS);
	}

	/* Do this before schema validation happens */

	/* fill in some defaults */
	name = XML_ATTR_GENERATION_ADMIN;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_warn("No value for %s was specified in the configuration.", name);
	crm_warn("The recommended course of action is to shutdown,"
	" run crm_verify and fix any errors it reports.");
	crm_warn("We will default to zero and continue but may get"
	" confused about which configuration to use if"
	" multiple nodes are powered up at the same time.");
	crm_xml_add_int(root, name, 0);
	}

	name = XML_ATTR_GENERATION;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_xml_add_int(root, name, 0);
	}

	name = XML_ATTR_NUMUPDATES;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_xml_add_int(root, name, 0);
	}

	// Unset (DC should set appropriate value)
	xml_remove_prop(root, XML_ATTR_DC_UUID);

	if (discard_status) {
	crm_log_xml_trace(root, "[on-disk]");
	}

	validation = crm_element_value(root, XML_ATTR_VALIDATION);
	if (validate_xml(root, NULL, TRUE) == FALSE) {
	crm_err("CIB does not validate with %s", crm_str(validation));
	cib_status = -pcmk_err_schema_validation;

	} else if (validation == NULL) {
	int version = 0;

	update_validation(&root, &version, 0, FALSE, FALSE);
	if (version > 0) {
	crm_notice("Enabling %s validation on"
	" the existing (sane) configuration", get_schema_name(version));
	} else {
	crm_err("CIB does not validate with any known schema");
	cib_status = -pcmk_err_schema_validation;
	}
	}

	return root;
	}

	gboolean
	uninitializeCib(void)
	{
	xmlNode *tmp_cib = the_cib;

	if (tmp_cib == NULL) {
	crm_debug("The CIB has already been deallocated.");
	return FALSE;
	}

	the_cib = NULL;

	crm_debug("Deallocating the CIB.");

	free_xml(tmp_cib);

	crm_debug("The CIB has been deallocated.");

	return TRUE;
	}

	/*
	* This method will free the old CIB pointer on success and the new one
	* on failure.
	*/
	int
	activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
	{
	if (new_cib) {
	xmlNode *saved_cib = the_cib;

	CRM_ASSERT(new_cib != saved_cib);
	the_cib = new_cib;
	free_xml(saved_cib);
	if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
	crm_debug("Triggering CIB write for %s op", op);
	mainloop_set_trigger(cib_writer);
	}
	return pcmk_ok;
	}

	crm_err("Ignoring invalid CIB");
	if (the_cib) {
	crm_warn("Reverting to last known CIB");
	} else {
	crm_crit("Could not write out new CIB and no saved version to revert to");
	}
	return -ENODATA;
	}

	static void
	cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
	{
	- if (signo) {
	- crm_notice("Disk write process terminated with signal %d (pid=%d, core=%d)", signo, pid,
	- core);
	+ const char *errmsg = "Could not write CIB to disk";

	- } else {
	- do_crm_log(exitcode == 0 ? LOG_TRACE : LOG_ERR, "Disk write process exited (pid=%d, rc=%d)",
	- pid, exitcode);
	+ if ((exitcode != 0) && cib_writes_enabled) {
	+ cib_writes_enabled = FALSE;
	+ errmsg = "Disabling CIB disk writes after failure";
	}

	- if (exitcode != 0 && cib_writes_enabled) {
	- crm_err("Disabling disk writes after write failure");
	- cib_writes_enabled = FALSE;
	+ if ((signo == 0) && (exitcode == 0)) {
	+ crm_trace("Disk write [%d] succeeded", (int) pid);
	+
	+ } else if (signo == 0) {
	+ crm_err("%s: process %d exited %d", errmsg, (int) pid, exitcode);
	+
	+ } else {
	+ crm_err("%s: process %d terminated with signal %d (%s)%s",
	+ errmsg, (int) pid, signo, strsignal(signo),
	+ (core? " and dumped core" : ""));
	}

	mainloop_trigger_complete(cib_writer);
	}

	int
	write_cib_contents(gpointer p)
	{
	int exit_rc = pcmk_ok;
	xmlNode *cib_local = NULL;

	/* Make a copy of the CIB to write (possibly in a forked child) */
	if (p) {
	/* Synchronous write out */
	cib_local = copy_xml(p);

	} else {
	int pid = 0;
	int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);

	/* Turn it off before the fork() to avoid:
	* - 2 processes writing to the same shared mem
	* - the child needing to disable it
	* (which would close it from underneath the parent)
	* This way, the shared mem files are already closed
	*/
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);

	pid = fork();
	if (pid < 0) {
	crm_perror(LOG_ERR, "Disabling disk writes after fork failure");
	cib_writes_enabled = FALSE;
	return FALSE;
	}

	if (pid) {
	/* Parent */
	mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
	if (bb_state == QB_LOG_STATE_ENABLED) {
	/* Re-enable now that it it safe */
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
	}

	return -1; /* -1 means 'still work to do' */
	}

	/* Asynchronous write-out after a fork() */

	/* In theory, we can scribble on the_cib here and not affect the parent,
	* but let's be safe anyway.
	*/
	cib_local = copy_xml(the_cib);
	}

	/* Write the CIB */
	exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml");

	/* A nonzero exit code will cause further writes to be disabled */
	free_xml(cib_local);
	if (p == NULL) {
	crm_exit_t exit_code = CRM_EX_OK;

	switch (exit_rc) {
	case pcmk_ok:
	exit_code = CRM_EX_OK;
	break;
	case pcmk_err_cib_modified:
	exit_code = CRM_EX_DIGEST; // Existing CIB doesn't match digest
	break;
	case pcmk_err_cib_backup: // Existing CIB couldn't be backed up
	case pcmk_err_cib_save: // New CIB couldn't be saved
	exit_code = CRM_EX_CANTCREAT;
	break;
	default:
	exit_code = CRM_EX_ERROR;
	break;
	}

	/* Use _exit() because exit() could affect the parent adversely */
	_exit(exit_code);
	}
	return exit_rc;
	}
	diff --git a/daemons/pacemakerd/pcmkd_subdaemons.c b/daemons/pacemakerd/pcmkd_subdaemons.c
	index b1aea780f2..9556d7409c 100644
	--- a/daemons/pacemakerd/pcmkd_subdaemons.c
	+++ b/daemons/pacemakerd/pcmkd_subdaemons.c
	@@ -1,840 +1,841 @@
	/*
	* Copyright 2010-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include "pacemakerd.h"

	#include <errno.h>
	#include <grp.h>
	#include <signal.h>
	#include <stdbool.h>
	#include <stdlib.h>
	#include <string.h>
	#include <sys/types.h>
	#include <time.h>
	#include <unistd.h>

	#include <crm/cluster.h>
	#include <crm/msg_xml.h>

	typedef struct pcmk_child_s {
	pid_t pid;
	int respawn_count;
	bool respawn;
	const char *name;
	const char *uid;
	const char *command;
	const char endpoint; / IPC server name */
	bool needs_cluster;

	/* Anything below here will be dynamically initialized */
	bool needs_retry;
	bool active_before_startup;
	} pcmk_child_t;

	#define PCMK_PROCESS_CHECK_INTERVAL 5
	#define SHUTDOWN_ESCALATION_PERIOD 180000 /* 3m */

	/* Index into the array below */
	#define PCMK_CHILD_CONTROLD 5

	static pcmk_child_t pcmk_children[] = {
	{
	0, 0, true, "pacemaker-based", CRM_DAEMON_USER,
	CRM_DAEMON_DIR "/pacemaker-based", PCMK__SERVER_BASED_RO,
	true
	},
	{
	0, 0, true, "pacemaker-fenced", NULL,
	CRM_DAEMON_DIR "/pacemaker-fenced", "stonith-ng",
	true
	},
	{
	0, 0, true, "pacemaker-execd", NULL,
	CRM_DAEMON_DIR "/pacemaker-execd", CRM_SYSTEM_LRMD,
	false
	},
	{
	0, 0, true, "pacemaker-attrd", CRM_DAEMON_USER,
	CRM_DAEMON_DIR "/pacemaker-attrd", T_ATTRD,
	true
	},
	{
	0, 0, true, "pacemaker-schedulerd", CRM_DAEMON_USER,
	CRM_DAEMON_DIR "/pacemaker-schedulerd", CRM_SYSTEM_PENGINE,
	false
	},
	{
	0, 0, true, "pacemaker-controld", CRM_DAEMON_USER,
	CRM_DAEMON_DIR "/pacemaker-controld", CRM_SYSTEM_CRMD,
	true
	},
	};

	static char *opts_default[] = { NULL, NULL };
	static char *opts_vgrind[] = { NULL, NULL, NULL, NULL, NULL };

	crm_trigger_t *shutdown_trigger = NULL;
	crm_trigger_t *startup_trigger = NULL;

	/* When contacted via pacemakerd-api by a client having sbd in
	* the name we assume it is sbd-daemon which wants to know
	* if pacemakerd shutdown gracefully.
	* Thus when everything is shutdown properly pacemakerd
	* waits till it has reported the graceful completion of
	* shutdown to sbd and just when sbd-client closes the
	* connection we can assume that the report has arrived
	* properly so that pacemakerd can finally exit.
	* Following two variables are used to track that handshake.
	*/
	unsigned int shutdown_complete_state_reported_to = 0;
	gboolean shutdown_complete_state_reported_client_closed = FALSE;

	/* state we report when asked via pacemakerd-api status-ping */
	const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
	gboolean running_with_sbd = FALSE; /* local copy */

	GMainLoop *mainloop = NULL;

	static gboolean fatal_error = FALSE;
	static bool global_keep_tracking = false;

	static gboolean check_active_before_startup_processes(gpointer user_data);
	static int child_liveness(pcmk_child_t *child);
	static gboolean escalate_shutdown(gpointer data);
	static int start_child(pcmk_child_t * child);
	static void pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode);
	static void pcmk_process_exit(pcmk_child_t * child);
	static gboolean pcmk_shutdown_worker(gpointer user_data);
	static gboolean stop_child(pcmk_child_t * child, int signal);

	static bool
	pcmkd_cluster_connected(void)
	{
	#if SUPPORT_COROSYNC
	return pcmkd_corosync_connected();
	#else
	return true;
	#endif
	}

	static gboolean
	check_active_before_startup_processes(gpointer user_data)
	{
	gboolean keep_tracking = FALSE;

	for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
	if (!pcmk_children[i].active_before_startup) {
	/* we are already tracking it as a child process. */
	continue;
	} else {
	int rc = child_liveness(&pcmk_children[i]);

	switch (rc) {
	case pcmk_rc_ok:
	break;
	case pcmk_rc_ipc_unresponsive:
	case pcmk_rc_ipc_pid_only: // This case: it was previously OK
	if (pcmk_children[i].respawn) {
	crm_err("%s[%lld] terminated%s", pcmk_children[i].name,
	(long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[i].pid),
	(rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
	} else {
	/* orderly shutdown */
	crm_notice("%s[%lld] terminated%s", pcmk_children[i].name,
	(long long) PCMK__SPECIAL_PID_AS_0(pcmk_children[i].pid),
	(rc == pcmk_rc_ipc_pid_only)? " as IPC server" : "");
	}
	pcmk_process_exit(&(pcmk_children[i]));
	continue;
	default:
	crm_exit(CRM_EX_FATAL);
	break; /* static analysis/noreturn */
	}
	}
	/* at least one of the processes found at startup
	* is still going, so keep this recurring timer around */
	keep_tracking = TRUE;
	}

	global_keep_tracking = keep_tracking;
	return keep_tracking;
	}

	static gboolean
	escalate_shutdown(gpointer data)
	{
	pcmk_child_t *child = data;

	if (child->pid == PCMK__SPECIAL_PID) {
	pcmk_process_exit(child);

	} else if (child->pid != 0) {
	/* Use SIGSEGV instead of SIGKILL to create a core so we can see what it was up to */
	crm_err("Child %s not terminating in a timely manner, forcing", child->name);
	stop_child(child, SIGSEGV);
	}
	return FALSE;
	}

	static void
	pcmk_child_exit(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
	{
	pcmk_child_t *child = mainloop_child_userdata(p);
	const char *name = mainloop_child_name(p);

	if (signo) {
	do_crm_log(((signo == SIGKILL)? LOG_WARNING : LOG_ERR),
	- "%s[%d] terminated with signal %d (core=%d)",
	- name, pid, signo, core);
	+ "%s[%d] terminated with signal %d (%s)%s",
	+ name, pid, signo, strsignal(signo),
	+ (core? " and dumped core" : ""));

	} else {
	switch(exitcode) {
	case CRM_EX_OK:
	crm_info("%s[%d] exited with status %d (%s)",
	name, pid, exitcode, crm_exit_str(exitcode));
	break;

	case CRM_EX_FATAL:
	crm_warn("Shutting cluster down because %s[%d] had fatal failure",
	name, pid);
	child->respawn = false;
	fatal_error = TRUE;
	pcmk_shutdown(SIGTERM);
	break;

	case CRM_EX_PANIC:
	crm_emerg("%s[%d] instructed the machine to reset", name, pid);
	child->respawn = false;
	fatal_error = TRUE;
	pcmk__panic(__func__);
	pcmk_shutdown(SIGTERM);
	break;

	default:
	crm_err("%s[%d] exited with status %d (%s)",
	name, pid, exitcode, crm_exit_str(exitcode));
	break;
	}
	}

	pcmk_process_exit(child);
	}

	static void
	pcmk_process_exit(pcmk_child_t * child)
	{
	child->pid = 0;
	child->active_before_startup = false;

	child->respawn_count += 1;
	if (child->respawn_count > MAX_RESPAWN) {
	crm_err("Child respawn count exceeded by %s", child->name);
	child->respawn = false;
	}

	if (shutdown_trigger) {
	/* resume step-wise shutdown (returned TRUE yields no parallelizing) */
	mainloop_set_trigger(shutdown_trigger);

	} else if (!child->respawn) {
	/* nothing to do */

	} else if (crm_is_true(getenv("PCMK_fail_fast"))) {
	crm_err("Rebooting system because of %s", child->name);
	pcmk__panic(__func__);

	} else if (child_liveness(child) == pcmk_rc_ok) {
	crm_warn("One-off suppressing strict respawning of a child process %s,"
	" appears alright per %s IPC end-point",
	child->name, child->endpoint);
	/* need to monitor how it evolves, and start new process if badly */
	child->active_before_startup = true;
	if (!global_keep_tracking) {
	global_keep_tracking = true;
	g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
	check_active_before_startup_processes, NULL);
	}

	} else {
	if (child->needs_cluster && !pcmkd_cluster_connected()) {
	crm_notice("Skipping cluster-based subdaemon %s until cluster returns",
	child->name);
	child->needs_retry = true;
	return;
	}

	crm_notice("Respawning failed child process: %s", child->name);
	start_child(child);
	}
	}

	static gboolean
	pcmk_shutdown_worker(gpointer user_data)
	{
	static int phase = PCMK__NELEM(pcmk_children) - 1;
	static time_t next_log = 0;

	if (phase == PCMK__NELEM(pcmk_children) - 1) {
	crm_notice("Shutting down Pacemaker");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
	}

	for (; phase >= 0; phase--) {
	pcmk_child_t *child = &(pcmk_children[phase]);

	if (child->pid != 0) {
	time_t now = time(NULL);

	if (child->respawn) {
	if (child->pid == PCMK__SPECIAL_PID) {
	crm_warn("The process behind %s IPC cannot be"
	" terminated, so either wait the graceful"
	" period of %ld s for its native termination"
	" if it vitally depends on some other daemons"
	" going down in a controlled way already,"
	" or locate and kill the correct %s process"
	" on your own; set PCMK_fail_fast=1 to avoid"
	" this altogether next time around",
	child->name, (long) SHUTDOWN_ESCALATION_PERIOD,
	child->command);
	}
	next_log = now + 30;
	child->respawn = false;
	stop_child(child, SIGTERM);
	if (phase < PCMK_CHILD_CONTROLD) {
	g_timeout_add(SHUTDOWN_ESCALATION_PERIOD,
	escalate_shutdown, child);
	}

	} else if (now >= next_log) {
	next_log = now + 30;
	crm_notice("Still waiting for %s to terminate "
	CRM_XS " pid=%lld",
	child->name, (long long) child->pid);
	}
	return TRUE;
	}

	/* cleanup */
	crm_debug("%s confirmed stopped", child->name);
	child->pid = 0;
	}

	crm_notice("Shutdown complete");
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
	if (!fatal_error && running_with_sbd &&
	pcmk__get_sbd_sync_resource_startup() &&
	!shutdown_complete_state_reported_client_closed) {
	crm_notice("Waiting for SBD to pick up shutdown-complete-state.");
	return TRUE;
	}

	{
	const char *delay = pcmk__env_option(PCMK__ENV_SHUTDOWN_DELAY);
	if(delay) {
	sync();
	pcmk__sleep_ms(crm_get_msec(delay));
	}
	}

	g_main_loop_quit(mainloop);

	if (fatal_error) {
	crm_notice("Shutting down and staying down after fatal error");
	#ifdef SUPPORT_COROSYNC
	pcmkd_shutdown_corosync();
	#endif
	crm_exit(CRM_EX_FATAL);
	}

	return TRUE;
	}

	/* TODO once libqb is taught to juggle with IPC end-points carried over as
	bare file descriptor (https://github.com/ClusterLabs/libqb/issues/325)
	it shall hand over these descriptors here if/once they are successfully
	pre-opened in (presumably) child_liveness(), to avoid any remaining
	room for races */
	// \return Standard Pacemaker return code
	static int
	start_child(pcmk_child_t * child)
	{
	uid_t uid = 0;
	gid_t gid = 0;
	gboolean use_valgrind = FALSE;
	gboolean use_callgrind = FALSE;
	const char *env_valgrind = getenv("PCMK_valgrind_enabled");
	const char *env_callgrind = getenv("PCMK_callgrind_enabled");

	child->active_before_startup = false;

	if (child->command == NULL) {
	crm_info("Nothing to do for child \"%s\"", child->name);
	return pcmk_rc_ok;
	}

	if (env_callgrind != NULL && crm_is_true(env_callgrind)) {
	use_callgrind = TRUE;
	use_valgrind = TRUE;

	} else if (env_callgrind != NULL && strstr(env_callgrind, child->name)) {
	use_callgrind = TRUE;
	use_valgrind = TRUE;

	} else if (env_valgrind != NULL && crm_is_true(env_valgrind)) {
	use_valgrind = TRUE;

	} else if (env_valgrind != NULL && strstr(env_valgrind, child->name)) {
	use_valgrind = TRUE;
	}

	if (use_valgrind && strlen(VALGRIND_BIN) == 0) {
	crm_warn("Cannot enable valgrind for %s:"
	" The location of the valgrind binary is unknown", child->name);
	use_valgrind = FALSE;
	}

	if (child->uid) {
	if (crm_user_lookup(child->uid, &uid, &gid) < 0) {
	crm_err("Invalid user (%s) for %s: not found", child->uid, child->name);
	return EACCES;
	}
	crm_info("Using uid=%u and group=%u for process %s", uid, gid, child->name);
	}

	child->pid = fork();
	CRM_ASSERT(child->pid != -1);

	if (child->pid > 0) {
	/* parent */
	mainloop_child_add(child->pid, 0, child->name, child, pcmk_child_exit);

	crm_info("Forked child %lld for process %s%s",
	(long long) child->pid, child->name,
	use_valgrind ? " (valgrind enabled: " VALGRIND_BIN ")" : "");
	return pcmk_rc_ok;

	} else {
	/* Start a new session */
	(void)setsid();

	/* Setup the two alternate arg arrays */
	opts_vgrind[0] = strdup(VALGRIND_BIN);
	if (use_callgrind) {
	opts_vgrind[1] = strdup("--tool=callgrind");
	opts_vgrind[2] = strdup("--callgrind-out-file=" CRM_STATE_DIR "/callgrind.out.%p");
	opts_vgrind[3] = strdup(child->command);
	opts_vgrind[4] = NULL;
	} else {
	opts_vgrind[1] = strdup(child->command);
	opts_vgrind[2] = NULL;
	opts_vgrind[3] = NULL;
	opts_vgrind[4] = NULL;
	}
	opts_default[0] = strdup(child->command);

	if(gid) {
	// Whether we need root group access to talk to cluster layer
	bool need_root_group = TRUE;

	if (is_corosync_cluster()) {
	/* Corosync clusters can drop root group access, because we set
	* uidgid.gid.${gid}=1 via CMAP, which allows these processes to
	* connect to corosync.
	*/
	need_root_group = FALSE;
	}

	// Drop root group access if not needed
	if (!need_root_group && (setgid(gid) < 0)) {
	crm_warn("Could not set group to %d: %s", gid, strerror(errno));
	}

	/* Initialize supplementary groups to only those always granted to
	* the user, plus haclient (so we can access IPC).
	*/
	if (initgroups(child->uid, gid) < 0) {
	crm_err("Cannot initialize groups for %s: %s (%d)", child->uid, pcmk_strerror(errno), errno);
	}
	}

	if (uid && setuid(uid) < 0) {
	crm_warn("Could not set user to %s (id %d): %s",
	child->uid, uid, strerror(errno));
	}

	pcmk__close_fds_in_child(true);

	pcmk__open_devnull(O_RDONLY); // stdin (fd 0)
	pcmk__open_devnull(O_WRONLY); // stdout (fd 1)
	pcmk__open_devnull(O_WRONLY); // stderr (fd 2)

	if (use_valgrind) {
	(void)execvp(VALGRIND_BIN, opts_vgrind);
	} else {
	(void)execvp(child->command, opts_default);
	}
	crm_crit("Could not execute %s: %s", child->command, strerror(errno));
	crm_exit(CRM_EX_FATAL);
	}
	return pcmk_rc_ok; /* never reached */
	}

	/*!
	* \internal
	* \brief Check the liveness of the child based on IPC name and PID if tracked
	*
	* \param[inout] child Child tracked data
	*
	* \return Standard Pacemaker return code
	*
	* \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
	* indicating that no trace of IPC liveness was detected,
	* pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
	* an unauthorized process, and pcmk_rc_ipc_pid_only indicating that
	* the child is up by PID but not IPC end-point (possibly starting).
	* \note This function doesn't modify any of \p child members but \c pid,
	* and is not actively toying with processes as such but invoking
	* \c stop_child in one particular case (there's for some reason
	* a different authentic holder of the IPC end-point).
	*/
	static int
	child_liveness(pcmk_child_t *child)
	{
	uid_t cl_uid = 0;
	gid_t cl_gid = 0;
	const uid_t root_uid = 0;
	const gid_t root_gid = 0;
	const uid_t *ref_uid;
	const gid_t *ref_gid;
	int rc = pcmk_rc_ipc_unresponsive;
	pid_t ipc_pid = 0;

	if (child->endpoint == NULL
	&& (child->pid <= 0 \|\| child->pid == PCMK__SPECIAL_PID)) {
	crm_err("Cannot track child %s for missing both API end-point and PID",
	child->name);
	rc = EINVAL; // Misuse of function when child is not trackable

	} else if (child->endpoint != NULL) {
	int legacy_rc = pcmk_ok;

	if (child->uid == NULL) {
	ref_uid = &root_uid;
	ref_gid = &root_gid;
	} else {
	ref_uid = &cl_uid;
	ref_gid = &cl_gid;
	legacy_rc = pcmk_daemon_user(&cl_uid, &cl_gid);
	}

	if (legacy_rc < 0) {
	rc = pcmk_legacy2rc(legacy_rc);
	crm_err("Could not find user and group IDs for user %s: %s "
	CRM_XS " rc=%d", CRM_DAEMON_USER, pcmk_rc_str(rc), rc);
	} else {
	rc = pcmk__ipc_is_authentic_process_active(child->endpoint,
	ref_uid, ref_gid,
	&ipc_pid);
	if ((rc == pcmk_rc_ok) \|\| (rc == pcmk_rc_ipc_unresponsive)) {
	if (child->pid <= 0) {
	/* If rc is pcmk_rc_ok, ipc_pid is nonzero and this
	* initializes a new child. If rc is
	* pcmk_rc_ipc_unresponsive, ipc_pid is zero, and we will
	* investigate further.
	*/
	child->pid = ipc_pid;
	} else if ((ipc_pid != 0) && (child->pid != ipc_pid)) {
	/* An unexpected (but authorized) process is responding to
	* IPC. Investigate further.
	*/
	rc = pcmk_rc_ipc_unresponsive;
	}
	}
	}
	}

	if (rc == pcmk_rc_ipc_unresponsive) {
	/* If we get here, a child without IPC is being tracked, no IPC liveness
	* has been detected, or IPC liveness has been detected with an
	* unexpected (but authorized) process. This is safe on FreeBSD since
	* the only change possible from a proper child's PID into "special" PID
	* of 1 behind more loosely related process.
	*/
	int ret = pcmk__pid_active(child->pid, child->name);

	if (ipc_pid && ((ret != pcmk_rc_ok)
	\|\| ipc_pid == PCMK__SPECIAL_PID
	\|\| (pcmk__pid_active(ipc_pid,
	child->name) == pcmk_rc_ok))) {
	/* An unexpected (but authorized) process was detected at the IPC
	* endpoint, and either it is active, or the child we're tracking is
	* not.
	*/

	if (ret == pcmk_rc_ok) {
	/* The child we're tracking is active. Kill it, and adopt the
	* detected process. This assumes that our children don't fork
	* (thus getting a different PID owning the IPC), but rather the
	* tracking got out of sync because of some means external to
	* Pacemaker, and adopting the detected process is better than
	* killing it and possibly having to spawn a new child.
	*/
	/* not possessing IPC, afterall (what about corosync CPG?) */
	stop_child(child, SIGKILL);
	}
	rc = pcmk_rc_ok;
	child->pid = ipc_pid;
	} else if (ret == pcmk_rc_ok) {
	// Our tracked child's PID was found active, but not its IPC
	rc = pcmk_rc_ipc_pid_only;
	} else if ((child->pid == 0) && (ret == EINVAL)) {
	// FreeBSD can return EINVAL
	rc = pcmk_rc_ipc_unresponsive;
	} else {
	switch (ret) {
	case EACCES:
	rc = pcmk_rc_ipc_unauthorized;
	break;
	case ESRCH:
	rc = pcmk_rc_ipc_unresponsive;
	break;
	default:
	rc = ret;
	break;
	}
	}
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Initial one-off check of the pre-existing "child" processes
	*
	* With "child" process, we mean the subdaemon that defines an API end-point
	* (all of them do as of the comment) -- the possible complement is skipped
	* as it is deemed it has no such shared resources to cause conflicts about,
	* hence it can presumably be started anew without hesitation.
	* If that won't hold true in the future, the concept of a shared resource
	* will have to be generalized beyond the API end-point.
	*
	* For boundary cases that the "child" is still starting (IPC end-point is yet
	* to be witnessed), or more rarely (practically FreeBSD only), when there's
	* a pre-existing "untrackable" authentic process, we give the situation some
	* time to possibly unfold in the right direction, meaning that said socket
	* will appear or the unattainable process will disappear per the observable
	* IPC, respectively.
	*
	* \return Standard Pacemaker return code
	*
	* \note Since this gets run at the very start, \c respawn_count fields
	* for particular children get temporarily overloaded with "rounds
	* of waiting" tracking, restored once we are about to finish with
	* success (i.e. returning value >=0) and will remain unrestored
	* otherwise. One way to suppress liveness detection logic for
	* particular child is to set the said value to a negative number.
	*/
	#define WAIT_TRIES 4 /* together with interleaved sleeps, worst case ~ 1s */
	int
	find_and_track_existing_processes(void)
	{
	bool tracking = false;
	bool wait_in_progress;
	int rc;
	size_t i, rounds;

	for (rounds = 1; rounds <= WAIT_TRIES; rounds++) {
	wait_in_progress = false;
	for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {

	if ((pcmk_children[i].endpoint == NULL)
	\|\| (pcmk_children[i].respawn_count < 0)) {
	continue;
	}

	rc = child_liveness(&pcmk_children[i]);
	if (rc == pcmk_rc_ipc_unresponsive) {
	/* As a speculation, don't give up if there are more rounds to
	* come for other reasons, but don't artificially wait just
	* because of this, since we would preferably start ASAP.
	*/
	continue;
	}

	pcmk_children[i].respawn_count = rounds;
	switch (rc) {
	case pcmk_rc_ok:
	if (pcmk_children[i].pid == PCMK__SPECIAL_PID) {
	if (crm_is_true(getenv("PCMK_fail_fast"))) {
	crm_crit("Cannot reliably track pre-existing"
	" authentic process behind %s IPC on this"
	" platform and PCMK_fail_fast requested",
	pcmk_children[i].endpoint);
	return EOPNOTSUPP;
	} else if (pcmk_children[i].respawn_count == WAIT_TRIES) {
	crm_notice("Assuming pre-existing authentic, though"
	" on this platform untrackable, process"
	" behind %s IPC is stable (was in %d"
	" previous samples) so rather than"
	" bailing out (PCMK_fail_fast not"
	" requested), we just switch to a less"
	" optimal IPC liveness monitoring"
	" (not very suitable for heavy load)",
	pcmk_children[i].name, WAIT_TRIES - 1);
	crm_warn("The process behind %s IPC cannot be"
	" terminated, so the overall shutdown"
	" will get delayed implicitly (%ld s),"
	" which serves as a graceful period for"
	" its native termination if it vitally"
	" depends on some other daemons going"
	" down in a controlled way already",
	pcmk_children[i].name,
	(long) SHUTDOWN_ESCALATION_PERIOD);
	} else {
	wait_in_progress = true;
	crm_warn("Cannot reliably track pre-existing"
	" authentic process behind %s IPC on this"
	" platform, can still disappear in %d"
	" attempt(s)", pcmk_children[i].endpoint,
	WAIT_TRIES - pcmk_children[i].respawn_count);
	continue;
	}
	}
	crm_notice("Tracking existing %s process (pid=%lld)",
	pcmk_children[i].name,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid));
	pcmk_children[i].respawn_count = -1; /* 0~keep watching */
	pcmk_children[i].active_before_startup = true;
	tracking = true;
	break;
	case pcmk_rc_ipc_pid_only:
	if (pcmk_children[i].respawn_count == WAIT_TRIES) {
	crm_crit("%s IPC end-point for existing authentic"
	" process %lld did not (re)appear",
	pcmk_children[i].endpoint,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid));
	return rc;
	}
	wait_in_progress = true;
	crm_warn("Cannot find %s IPC end-point for existing"
	" authentic process %lld, can still (re)appear"
	" in %d attempts (?)",
	pcmk_children[i].endpoint,
	(long long) PCMK__SPECIAL_PID_AS_0(
	pcmk_children[i].pid),
	WAIT_TRIES - pcmk_children[i].respawn_count);
	continue;
	default:
	crm_crit("Checked liveness of %s: %s " CRM_XS " rc=%d",
	pcmk_children[i].name, pcmk_rc_str(rc), rc);
	return rc;
	}
	}
	if (!wait_in_progress) {
	break;
	}
	pcmk__sleep_ms(250); // Wait a bit for changes to possibly happen
	}
	for (i = 0; i < PCMK__NELEM(pcmk_children); i++) {
	pcmk_children[i].respawn_count = 0; /* restore pristine state */
	}

	if (tracking) {
	g_timeout_add_seconds(PCMK_PROCESS_CHECK_INTERVAL,
	check_active_before_startup_processes, NULL);
	}
	return pcmk_rc_ok;
	}

	gboolean
	init_children_processes(void *user_data)
	{
	/* start any children that have not been detected */
	for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
	if (pcmk_children[i].pid != 0) {
	/* we are already tracking it */
	continue;
	}

	start_child(&(pcmk_children[i]));
	}

	/* From this point on, any daemons being started will be due to
	* respawning rather than node start.
	*
	* This may be useful for the daemons to know
	*/
	setenv("PCMK_respawned", "true", 1);
	pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
	return TRUE;
	}

	void
	pcmk_shutdown(int nsig)
	{
	if (shutdown_trigger == NULL) {
	shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL);
	}
	mainloop_set_trigger(shutdown_trigger);
	}

	void
	restart_cluster_subdaemons(void)
	{
	for (int i = 0; i < PCMK__NELEM(pcmk_children); i++) {
	if (!pcmk_children[i].needs_retry \|\| pcmk_children[i].pid != 0) {
	continue;
	}

	crm_notice("Respawning cluster-based subdaemon: %s", pcmk_children[i].name);
	if (start_child(&pcmk_children[i])) {
	pcmk_children[i].needs_retry = false;
	}
	}
	}

	static gboolean
	stop_child(pcmk_child_t * child, int signal)
	{
	if (signal == 0) {
	signal = SIGTERM;
	}

	/* why to skip PID of 1?
	- FreeBSD ~ how untrackable process behind IPC is masqueraded as
	- elsewhere: how "init" task is designated; in particular, in systemd
	arrangement of socket-based activation, this is pretty real */
	if (child->command == NULL \|\| child->pid == PCMK__SPECIAL_PID) {
	crm_debug("Nothing to do for child \"%s\" (process %lld)",
	child->name, (long long) PCMK__SPECIAL_PID_AS_0(child->pid));
	return TRUE;
	}

	if (child->pid <= 0) {
	crm_trace("Client %s not running", child->name);
	return TRUE;
	}

	errno = 0;
	if (kill(child->pid, signal) == 0) {
	crm_notice("Stopping %s "CRM_XS" sent signal %d to process %lld",
	child->name, signal, (long long) child->pid);

	} else {
	crm_err("Could not stop %s (process %lld) with signal %d: %s",
	child->name, (long long) child->pid, signal, strerror(errno));
	}

	return TRUE;
	}

	diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
	index 9a2b16bc67..fc3e2cb576 100644
	--- a/lib/common/mainloop.c
	+++ b/lib/common/mainloop.c
	@@ -1,1502 +1,1502 @@
	/*
	* Copyright 2004-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <stdlib.h>
	#include <string.h>
	#include <signal.h>
	#include <errno.h>

	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc_internal.h>

	#include <qb/qbarray.h>

	struct mainloop_child_s {
	pid_t pid;
	char *desc;
	unsigned timerid;
	gboolean timeout;
	void *privatedata;

	enum mainloop_child_flags flags;

	/* Called when a process dies */
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode);
	};

	struct trigger_s {
	GSource source;
	gboolean running;
	gboolean trigger;
	void *user_data;
	guint id;

	};

	struct mainloop_timer_s {
	guint id;
	guint period_ms;
	bool repeat;
	char *name;
	GSourceFunc cb;
	void *userdata;
	};

	static gboolean
	crm_trigger_prepare(GSource * source, gint * timeout)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	/* cluster-glue's FD and IPC related sources make use of
	* g_source_add_poll() but do not set a timeout in their prepare
	* functions
	*
	* This means mainloop's poll() will block until an event for one
	* of these sources occurs - any /other/ type of source, such as
	* this one or g_idle_*, that doesn't use g_source_add_poll() is
	* S-O-L and won't be processed until there is something fd-based
	* happens.
	*
	* Luckily the timeout we can set here affects all sources and
	* puts an upper limit on how long poll() can take.
	*
	* So unconditionally set a small-ish timeout, not too small that
	* we're in constant motion, which will act as an upper bound on
	* how long the signal handling might be delayed for.
	*/
	timeout = 500; / Timeout in ms */

	return trig->trigger;
	}

	static gboolean
	crm_trigger_check(GSource * source)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	return trig->trigger;
	}

	/*!
	* \internal
	* \brief GSource dispatch function for crm_trigger_t
	*
	* \param[in] source crm_trigger_t being dispatched
	* \param[in] callback Callback passed at source creation
	* \param[in] userdata User data passed at source creation
	*
	* \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
	*/
	static gboolean
	crm_trigger_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	gboolean rc = G_SOURCE_CONTINUE;
	crm_trigger_t trig = (crm_trigger_t ) source;

	if (trig->running) {
	/* Wait until the existing job is complete before starting the next one */
	return G_SOURCE_CONTINUE;
	}
	trig->trigger = FALSE;

	if (callback) {
	int callback_rc = callback(trig->user_data);

	if (callback_rc < 0) {
	crm_trace("Trigger handler %p not yet complete", trig);
	trig->running = TRUE;
	} else if (callback_rc == 0) {
	rc = G_SOURCE_REMOVE;
	}
	}
	return rc;
	}

	static void
	crm_trigger_finalize(GSource * source)
	{
	crm_trace("Trigger %p destroyed", source);
	}

	static GSourceFuncs crm_trigger_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_trigger_dispatch,
	crm_trigger_finalize,
	};

	static crm_trigger_t *
	mainloop_setup_trigger(GSource * source, int priority, int (*dispatch) (gpointer user_data),
	gpointer userdata)
	{
	crm_trigger_t *trigger = NULL;

	trigger = (crm_trigger_t *) source;

	trigger->id = 0;
	trigger->trigger = FALSE;
	trigger->user_data = userdata;

	if (dispatch) {
	g_source_set_callback(source, dispatch, trigger, NULL);
	}

	g_source_set_priority(source, priority);
	g_source_set_can_recurse(source, FALSE);

	trigger->id = g_source_attach(source, NULL);
	return trigger;
	}

	void
	mainloop_trigger_complete(crm_trigger_t * trig)
	{
	crm_trace("Trigger handler %p complete", trig);
	trig->running = FALSE;
	}

	/*!
	* \brief Create a trigger to be used as a mainloop source
	*
	* \param[in] priority Relative priority of source (lower number is higher priority)
	* \param[in] dispatch Trigger dispatch function (should return 0 to remove the
	* trigger from the mainloop, -1 if the trigger should be
	* kept but the job is still running and not complete, and
	* 1 if the trigger should be kept and the job is complete)
	*
	* \return Newly allocated mainloop source for trigger
	*/
	crm_trigger_t *
	mainloop_add_trigger(int priority, int (*dispatch) (gpointer user_data), gpointer userdata)
	{
	GSource *source = NULL;

	CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
	source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
	CRM_ASSERT(source != NULL);

	return mainloop_setup_trigger(source, priority, dispatch, userdata);
	}

	void
	mainloop_set_trigger(crm_trigger_t * source)
	{
	if(source) {
	source->trigger = TRUE;
	}
	}

	gboolean
	mainloop_destroy_trigger(crm_trigger_t * source)
	{
	GSource *gs = NULL;

	if(source == NULL) {
	return TRUE;
	}

	gs = (GSource *)source;

	g_source_destroy(gs); /* Remove from mainloop, ref_count-- */
	g_source_unref(gs); /* The caller no longer carries a reference to source
	*
	* At this point the source should be free'd,
	* unless we're currently processing said
	* source, in which case mainloop holds an
	* additional reference and it will be free'd
	* once our processing completes
	*/
	return TRUE;
	}

	// Define a custom glib source for signal handling

	// Data structure for custom glib source
	typedef struct signal_s {
	crm_trigger_t trigger; // trigger that invoked source (must be first)
	void (*handler) (int sig); // signal handler
	int signal; // signal that was received
	} crm_signal_t;

	// Table to associate signal handlers with signal numbers
	static crm_signal_t *crm_signals[NSIG];

	/*!
	* \internal
	* \brief Dispatch an event from custom glib source for signals
	*
	* Given an signal event, clear the event trigger and call any registered
	* signal handler.
	*
	* \param[in] source glib source that triggered this dispatch
	* \param[in] callback (ignored)
	* \param[in] userdata (ignored)
	*/
	static gboolean
	crm_signal_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	crm_signal_t sig = (crm_signal_t ) source;

	if(sig->signal != SIGCHLD) {
	crm_notice("Caught '%s' signal "CRM_XS" %d (%s handler)",
	strsignal(sig->signal), sig->signal,
	(sig->handler? "invoking" : "no"));
	}

	sig->trigger.trigger = FALSE;
	if (sig->handler) {
	sig->handler(sig->signal);
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Handle a signal by setting a trigger for signal source
	*
	* \param[in] sig Signal number that was received
	*
	* \note This is the true signal handler for the mainloop signal source, and
	* must be async-safe.
	*/
	static void
	mainloop_signal_handler(int sig)
	{
	if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
	mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
	}
	}

	// Functions implementing our custom glib source for signal handling
	static GSourceFuncs crm_signal_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_signal_dispatch,
	crm_trigger_finalize,
	};

	/*!
	* \internal
	* \brief Set a true signal handler
	*
	* signal()-like interface to sigaction()
	*
	* \param[in] sig Signal number to register handler for
	* \param[in] dispatch Signal handler
	*
	* \return The previous value of the signal handler, or SIG_ERR on error
	* \note The dispatch function must be async-safe.
	*/
	sighandler_t
	crm_signal_handler(int sig, sighandler_t dispatch)
	{
	sigset_t mask;
	struct sigaction sa;
	struct sigaction old;

	if (sigemptyset(&mask) < 0) {
	crm_err("Could not set handler for signal %d: %s",
	sig, pcmk_strerror(errno));
	return SIG_ERR;
	}

	memset(&sa, 0, sizeof(struct sigaction));
	sa.sa_handler = dispatch;
	sa.sa_flags = SA_RESTART;
	sa.sa_mask = mask;

	if (sigaction(sig, &sa, &old) < 0) {
	crm_err("Could not set handler for signal %d: %s",
	sig, pcmk_strerror(errno));
	return SIG_ERR;
	}
	return old.sa_handler;
	}

	static void
	mainloop_destroy_signal_entry(int sig)
	{
	crm_signal_t *tmp = crm_signals[sig];

	crm_signals[sig] = NULL;

	crm_trace("Destroying signal %d", sig);
	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	}

	/*!
	* \internal
	* \brief Add a signal handler to a mainloop
	*
	* \param[in] sig Signal number to handle
	* \param[in] dispatch Signal handler function
	*
	* \note The true signal handler merely sets a mainloop trigger to call this
	* dispatch function via the mainloop. Therefore, the dispatch function
	* does not need to be async-safe.
	*/
	gboolean
	mainloop_add_signal(int sig, void (*dispatch) (int sig))
	{
	GSource *source = NULL;
	int priority = G_PRIORITY_HIGH - 1;

	if (sig == SIGTERM) {
	/* TERM is higher priority than other signals,
	* signals are higher priority than other ipc.
	* Yes, minus: smaller is "higher"
	*/
	priority--;
	}

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signals[sig] != NULL && crm_signals[sig]->handler == dispatch) {
	crm_trace("Signal handler for %d is already installed", sig);
	return TRUE;

	} else if (crm_signals[sig] != NULL) {
	crm_err("Different signal handler for %d is already installed", sig);
	return FALSE;
	}

	CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
	source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));

	crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
	CRM_ASSERT(crm_signals[sig] != NULL);

	crm_signals[sig]->handler = dispatch;
	crm_signals[sig]->signal = sig;

	if (crm_signal_handler(sig, mainloop_signal_handler) == SIG_ERR) {
	mainloop_destroy_signal_entry(sig);
	return FALSE;
	}
	#if 0
	/* If we want signals to interrupt mainloop's poll(), instead of waiting for
	* the timeout, then we should call siginterrupt() below
	*
	* For now, just enforce a low timeout
	*/
	if (siginterrupt(sig, 1) < 0) {
	crm_perror(LOG_INFO, "Could not enable system call interruptions for signal %d", sig);
	}
	#endif

	return TRUE;
	}

	gboolean
	mainloop_destroy_signal(int sig)
	{
	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signal_handler(sig, NULL) == SIG_ERR) {
	crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
	return FALSE;

	} else if (crm_signals[sig] == NULL) {
	return TRUE;
	}
	mainloop_destroy_signal_entry(sig);
	return TRUE;
	}

	static qb_array_t *gio_map = NULL;

	void
	mainloop_cleanup(void)
	{
	if (gio_map) {
	qb_array_free(gio_map);
	}

	for (int sig = 0; sig < NSIG; ++sig) {
	mainloop_destroy_signal_entry(sig);
	}
	}

	/*
	* libqb...
	*/
	struct gio_to_qb_poll {
	int32_t is_used;
	guint source;
	int32_t events;
	void *data;
	qb_ipcs_dispatch_fn_t fn;
	enum qb_loop_priority p;
	};

	static gboolean
	gio_read_socket(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;
	gint fd = g_io_channel_unix_get_fd(gio);

	crm_trace("%p.%d %d", data, fd, condition);

	/* if this assert get's hit, then there is a race condition between
	* when we destroy a fd and when mainloop actually gives it up */
	CRM_ASSERT(adaptor->is_used > 0);

	return (adaptor->fn(fd, condition, adaptor->data) == 0);
	}

	static void
	gio_poll_destroy(gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;

	adaptor->is_used--;
	CRM_ASSERT(adaptor->is_used >= 0);

	if (adaptor->is_used == 0) {
	crm_trace("Marking adaptor %p unused", adaptor);
	adaptor->source = 0;
	}
	}

	/*!
	* \internal
	* \brief Convert libqb's poll priority into GLib's one
	*
	* \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback)
	*
	* \return best matching GLib's priority
	*/
	static gint
	conv_prio_libqb2glib(enum qb_loop_priority prio)
	{
	gint ret = G_PRIORITY_DEFAULT;
	switch (prio) {
	case QB_LOOP_LOW:
	ret = G_PRIORITY_LOW;
	break;
	case QB_LOOP_HIGH:
	ret = G_PRIORITY_HIGH;
	break;
	default:
	crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED",
	prio);
	/* fall-through */
	case QB_LOOP_MED:
	break;
	}
	return ret;
	}

	/*!
	* \internal
	* \brief Convert libqb's poll priority to rate limiting spec
	*
	* \param[in] prio libqb's poll priority (#QB_LOOP_MED assumed as fallback)
	*
	* \return best matching rate limiting spec
	*/
	static enum qb_ipcs_rate_limit
	conv_libqb_prio2ratelimit(enum qb_loop_priority prio)
	{
	/* this is an inversion of what libqb's qb_ipcs_request_rate_limit does */
	enum qb_ipcs_rate_limit ret = QB_IPCS_RATE_NORMAL;
	switch (prio) {
	case QB_LOOP_LOW:
	ret = QB_IPCS_RATE_SLOW;
	break;
	case QB_LOOP_HIGH:
	ret = QB_IPCS_RATE_FAST;
	break;
	default:
	crm_trace("Invalid libqb's loop priority %d, assuming QB_LOOP_MED",
	prio);
	/* fall-through */
	case QB_LOOP_MED:
	break;
	}
	return ret;
	}

	static int32_t
	gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn, int32_t add)
	{
	struct gio_to_qb_poll *adaptor;
	GIOChannel *channel;
	int32_t res = 0;

	res = qb_array_index(gio_map, fd, (void **)&adaptor);
	if (res < 0) {
	crm_err("Array lookup failed for fd=%d: %d", fd, res);
	return res;
	}

	crm_trace("Adding fd=%d to mainloop as adaptor %p", fd, adaptor);

	if (add && adaptor->source) {
	crm_err("Adaptor for descriptor %d is still in-use", fd);
	return -EEXIST;
	}
	if (!add && !adaptor->is_used) {
	crm_err("Adaptor for descriptor %d is not in-use", fd);
	return -ENOENT;
	}

	/* channel is created with ref_count = 1 */
	channel = g_io_channel_unix_new(fd);
	if (!channel) {
	crm_err("No memory left to add fd=%d", fd);
	return -ENOMEM;
	}

	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}

	/* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
	evts \|= (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR);

	adaptor->fn = fn;
	adaptor->events = evts;
	adaptor->data = data;
	adaptor->p = p;
	adaptor->is_used++;
	adaptor->source =
	g_io_add_watch_full(channel, conv_prio_libqb2glib(p), evts,
	gio_read_socket, adaptor, gio_poll_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after gio_poll_destroy() completes
	*/
	g_io_channel_unref(channel);

	crm_trace("Added to mainloop with gsource id=%d", adaptor->source);
	if (adaptor->source > 0) {
	return 0;
	}

	return -EINVAL;
	}

	static int32_t
	gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_TRUE);
	}

	static int32_t
	gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_FALSE);
	}

	static int32_t
	gio_poll_dispatch_del(int32_t fd)
	{
	struct gio_to_qb_poll *adaptor;

	crm_trace("Looking for fd=%d", fd);
	if (qb_array_index(gio_map, fd, (void **)&adaptor) == 0) {
	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}
	}
	return 0;
	}

	struct qb_ipcs_poll_handlers gio_poll_funcs = {
	.job_add = NULL,
	.dispatch_add = gio_poll_dispatch_add,
	.dispatch_mod = gio_poll_dispatch_mod,
	.dispatch_del = gio_poll_dispatch_del,
	};

	static enum qb_ipc_type
	pick_ipc_type(enum qb_ipc_type requested)
	{
	const char *env = getenv("PCMK_ipc_type");

	if (env && strcmp("shared-mem", env) == 0) {
	return QB_IPC_SHM;
	} else if (env && strcmp("socket", env) == 0) {
	return QB_IPC_SOCKET;
	} else if (env && strcmp("posix", env) == 0) {
	return QB_IPC_POSIX_MQ;
	} else if (env && strcmp("sysv", env) == 0) {
	return QB_IPC_SYSV_MQ;
	} else if (requested == QB_IPC_NATIVE) {
	/* We prefer shared memory because the server never blocks on
	* send. If part of a message fits into the socket, libqb
	* needs to block until the remainder can be sent also.
	* Otherwise the client will wait forever for the remaining
	* bytes.
	*/
	return QB_IPC_SHM;
	}
	return requested;
	}

	qb_ipcs_service_t *
	mainloop_add_ipc_server(const char *name, enum qb_ipc_type type,
	struct qb_ipcs_service_handlers *callbacks)
	{
	return mainloop_add_ipc_server_with_prio(name, type, callbacks, QB_LOOP_MED);
	}

	qb_ipcs_service_t *
	mainloop_add_ipc_server_with_prio(const char *name, enum qb_ipc_type type,
	struct qb_ipcs_service_handlers *callbacks,
	enum qb_loop_priority prio)
	{
	int rc = 0;
	qb_ipcs_service_t *server = NULL;

	if (gio_map == NULL) {
	gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
	}

	server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);

	if (server == NULL) {
	crm_err("Could not create %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
	return NULL;
	}

	if (prio != QB_LOOP_MED) {
	qb_ipcs_request_rate_limit(server, conv_libqb_prio2ratelimit(prio));
	}

	/* All clients should use at least ipc_buffer_max as their buffer size */
	qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size());
	qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);

	rc = qb_ipcs_run(server);
	if (rc < 0) {
	crm_err("Could not start %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
	return NULL;
	}

	return server;
	}

	void
	mainloop_del_ipc_server(qb_ipcs_service_t * server)
	{
	if (server) {
	qb_ipcs_destroy(server);
	}
	}

	struct mainloop_io_s {
	char *name;
	void *userdata;

	int fd;
	guint source;
	crm_ipc_t *ipc;
	GIOChannel *channel;

	int (dispatch_fn_ipc) (const char buffer, ssize_t length, gpointer userdata);
	int (*dispatch_fn_io) (gpointer userdata);
	void (*destroy_fn) (gpointer userdata);

	};

	/*!
	* \internal
	* \brief I/O watch callback function (GIOFunc)
	*
	* \param[in] gio I/O channel being watched
	* \param[in] condition I/O condition satisfied
	* \param[in] data User data passed when source was created
	*
	* \return G_SOURCE_REMOVE to remove source, G_SOURCE_CONTINUE to keep it
	*/
	static gboolean
	mainloop_gio_callback(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	gboolean rc = G_SOURCE_CONTINUE;
	mainloop_io_t *client = data;

	CRM_ASSERT(client->fd == g_io_channel_unix_get_fd(gio));

	if (condition & G_IO_IN) {
	if (client->ipc) {
	long read_rc = 0L;
	int max = 10;

	do {
	read_rc = crm_ipc_read(client->ipc);
	if (read_rc <= 0) {
	crm_trace("Could not read IPC message from %s: %s (%ld)",
	client->name, pcmk_strerror(read_rc), read_rc);

	} else if (client->dispatch_fn_ipc) {
	const char *buffer = crm_ipc_buffer(client->ipc);

	crm_trace("New %ld-byte IPC message from %s "
	"after I/O condition %d",
	read_rc, client->name, (int) condition);
	if (client->dispatch_fn_ipc(buffer, read_rc, client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	rc = G_SOURCE_REMOVE;
	}
	}

	} while ((rc == G_SOURCE_CONTINUE) && (read_rc > 0) && --max > 0);

	} else {
	crm_trace("New I/O event for %s after I/O condition %d",
	client->name, (int) condition);
	if (client->dispatch_fn_io) {
	if (client->dispatch_fn_io(client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	rc = G_SOURCE_REMOVE;
	}
	}
	}
	}

	if (client->ipc && crm_ipc_connected(client->ipc) == FALSE) {
	crm_err("Connection to %s closed " CRM_XS "client=%p condition=%d",
	client->name, client, condition);
	rc = G_SOURCE_REMOVE;

	} else if (condition & (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR)) {
	crm_trace("The connection %s[%p] has been closed (I/O condition=%d)",
	client->name, client, condition);
	rc = G_SOURCE_REMOVE;

	} else if ((condition & G_IO_IN) == 0) {
	/*
	#define GLIB_SYSDEF_POLLIN =1
	#define GLIB_SYSDEF_POLLPRI =2
	#define GLIB_SYSDEF_POLLOUT =4
	#define GLIB_SYSDEF_POLLERR =8
	#define GLIB_SYSDEF_POLLHUP =16
	#define GLIB_SYSDEF_POLLNVAL =32

	typedef enum
	{
	G_IO_IN GLIB_SYSDEF_POLLIN,
	G_IO_OUT GLIB_SYSDEF_POLLOUT,
	G_IO_PRI GLIB_SYSDEF_POLLPRI,
	G_IO_ERR GLIB_SYSDEF_POLLERR,
	G_IO_HUP GLIB_SYSDEF_POLLHUP,
	G_IO_NVAL GLIB_SYSDEF_POLLNVAL
	} GIOCondition;

	A bitwise combination representing a condition to watch for on an event source.

	G_IO_IN There is data to read.
	G_IO_OUT Data can be written (without blocking).
	G_IO_PRI There is urgent data to read.
	G_IO_ERR Error condition.
	G_IO_HUP Hung up (the connection has been broken, usually for pipes and sockets).
	G_IO_NVAL Invalid request. The file descriptor is not open.
	*/
	crm_err("Strange condition: %d", condition);
	}

	/* G_SOURCE_REMOVE results in mainloop_gio_destroy() being called
	* just before the source is removed from mainloop
	*/
	return rc;
	}

	static void
	mainloop_gio_destroy(gpointer c)
	{
	mainloop_io_t *client = c;
	char *c_name = strdup(client->name);

	/* client->source is valid but about to be destroyed (ref_count == 0) in gmain.c
	* client->channel will still have ref_count > 0... should be == 1
	*/
	crm_trace("Destroying client %s[%p]", c_name, c);

	if (client->ipc) {
	crm_ipc_close(client->ipc);
	}

	if (client->destroy_fn) {
	void (*destroy_fn) (gpointer userdata) = client->destroy_fn;

	client->destroy_fn = NULL;
	destroy_fn(client->userdata);
	}

	if (client->ipc) {
	crm_ipc_t *ipc = client->ipc;

	client->ipc = NULL;
	crm_ipc_destroy(ipc);
	}

	crm_trace("Destroyed client %s[%p]", c_name, c);

	free(client->name); client->name = NULL;
	free(client);

	free(c_name);
	}

	/*!
	* \brief Connect to IPC and add it as a main loop source
	*
	* \param[in] ipc IPC connection to add
	* \param[in] priority Event source priority to use for connection
	* \param[in] userdata Data to register with callbacks
	* \param[in] callbacks Dispatch and destroy callbacks for connection
	* \param[out] source Newly allocated event source
	*
	* \return Standard Pacemaker return code
	*
	* \note On failure, the caller is still responsible for ipc. On success, the
	* caller should call mainloop_del_ipc_client() when source is no longer
	* needed, which will lead to the disconnection of the IPC later in the
	* main loop if it is connected. However the IPC disconnects,
	* mainloop_gio_destroy() will free ipc and source after calling the
	* destroy callback.
	*/
	int
	pcmk__add_mainloop_ipc(crm_ipc_t ipc, int priority, void userdata,
	struct ipc_client_callbacks *callbacks,
	mainloop_io_t **source)
	{
	CRM_CHECK((ipc != NULL) && (callbacks != NULL), return EINVAL);

	if (!crm_ipc_connect(ipc)) {
	int rc = errno;
	crm_debug("Connection to %s failed: %d", crm_ipc_name(ipc), errno);
	return rc;
	}
	*source = mainloop_add_fd(crm_ipc_name(ipc), priority, crm_ipc_get_fd(ipc),
	userdata, NULL);
	if (*source == NULL) {
	int rc = errno;

	crm_ipc_close(ipc);
	return rc;
	}
	(*source)->ipc = ipc;
	(*source)->destroy_fn = callbacks->destroy;
	(*source)->dispatch_fn_ipc = callbacks->dispatch;
	return pcmk_rc_ok;
	}

	/*!
	* \brief Get period for mainloop timer
	*
	* \param[in] timer Timer
	*
	* \return Period in ms
	*/
	guint
	pcmk__mainloop_timer_get_period(mainloop_timer_t *timer)
	{
	if (timer) {
	return timer->period_ms;
	}
	return 0;
	}

	mainloop_io_t *
	mainloop_add_ipc_client(const char *name, int priority, size_t max_size,
	void userdata, struct ipc_client_callbacks callbacks)
	{
	crm_ipc_t *ipc = crm_ipc_new(name, max_size);
	mainloop_io_t *source = NULL;
	int rc = pcmk__add_mainloop_ipc(ipc, priority, userdata, callbacks,
	&source);

	if (rc != pcmk_rc_ok) {
	if (crm_log_level == LOG_STDOUT) {
	fprintf(stderr, "Connection to %s failed: %s",
	name, pcmk_rc_str(rc));
	}
	crm_ipc_destroy(ipc);
	if (rc > 0) {
	errno = rc;
	} else {
	errno = ENOTCONN;
	}
	return NULL;
	}
	return source;
	}

	void
	mainloop_del_ipc_client(mainloop_io_t * client)
	{
	mainloop_del_fd(client);
	}

	crm_ipc_t *
	mainloop_get_ipc_client(mainloop_io_t * client)
	{
	if (client) {
	return client->ipc;
	}
	return NULL;
	}

	mainloop_io_t *
	mainloop_add_fd(const char name, int priority, int fd, void userdata,
	struct mainloop_fd_callbacks * callbacks)
	{
	mainloop_io_t *client = NULL;

	if (fd >= 0) {
	client = calloc(1, sizeof(mainloop_io_t));
	if (client == NULL) {
	return NULL;
	}
	client->name = strdup(name);
	client->userdata = userdata;

	if (callbacks) {
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_io = callbacks->dispatch;
	}

	client->fd = fd;
	client->channel = g_io_channel_unix_new(fd);
	client->source =
	g_io_add_watch_full(client->channel, priority,
	(G_IO_IN \| G_IO_HUP \| G_IO_NVAL \| G_IO_ERR), mainloop_gio_callback,
	client, mainloop_gio_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch() or g_source_remove()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after mainloop_gio_destroy() completes
	*/
	g_io_channel_unref(client->channel);
	crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
	} else {
	errno = EINVAL;
	}

	return client;
	}

	void
	mainloop_del_fd(mainloop_io_t * client)
	{
	if (client != NULL) {
	crm_trace("Removing client %s[%p]", client->name, client);
	if (client->source) {
	/* Results in mainloop_gio_destroy() being called just
	* before the source is removed from mainloop
	*/
	g_source_remove(client->source);
	}
	}
	}

	static GList *child_list = NULL;

	pid_t
	mainloop_child_pid(mainloop_child_t * child)
	{
	return child->pid;
	}

	const char *
	mainloop_child_name(mainloop_child_t * child)
	{
	return child->desc;
	}

	int
	mainloop_child_timeout(mainloop_child_t * child)
	{
	return child->timeout;
	}

	void *
	mainloop_child_userdata(mainloop_child_t * child)
	{
	return child->privatedata;
	}

	void
	mainloop_clear_child_userdata(mainloop_child_t * child)
	{
	child->privatedata = NULL;
	}

	/* good function name */
	static void
	child_free(mainloop_child_t *child)
	{
	if (child->timerid != 0) {
	crm_trace("Removing timer %d", child->timerid);
	g_source_remove(child->timerid);
	child->timerid = 0;
	}
	free(child->desc);
	free(child);
	}

	/* terrible function name */
	static int
	child_kill_helper(mainloop_child_t *child)
	{
	int rc;
	if (child->flags & mainloop_leave_pid_group) {
	crm_debug("Kill pid %d only. leave group intact.", child->pid);
	rc = kill(child->pid, SIGKILL);
	} else {
	crm_debug("Kill pid %d's group", child->pid);
	rc = kill(-child->pid, SIGKILL);
	}

	if (rc < 0) {
	if (errno != ESRCH) {
	crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
	}
	return -errno;
	}
	return 0;
	}

	static gboolean
	child_timeout_callback(gpointer p)
	{
	mainloop_child_t *child = p;
	int rc = 0;

	child->timerid = 0;
	if (child->timeout) {
	- crm_crit("%s process (PID %d) will not die!", child->desc, (int)child->pid);
	+ crm_warn("%s process (PID %d) will not die!", child->desc, (int)child->pid);
	return FALSE;
	}

	rc = child_kill_helper(child);
	if (rc == -ESRCH) {
	/* Nothing left to do. pid doesn't exist */
	return FALSE;
	}

	child->timeout = TRUE;
	- crm_warn("%s process (PID %d) timed out", child->desc, (int)child->pid);
	+ crm_debug("%s process (PID %d) timed out", child->desc, (int)child->pid);

	child->timerid = g_timeout_add(5000, child_timeout_callback, child);
	return FALSE;
	}

	static bool
	child_waitpid(mainloop_child_t *child, int flags)
	{
	int rc = 0;
	int core = 0;
	int signo = 0;
	int status = 0;
	int exitcode = 0;
	bool callback_needed = true;

	rc = waitpid(child->pid, &status, flags);
	if (rc == 0) { // WNOHANG in flags, and child status is not available
	crm_trace("Child process %d (%s) still active",
	child->pid, child->desc);
	callback_needed = false;

	} else if (rc != child->pid) {
	/* According to POSIX, possible conditions:
	* - child->pid was non-positive (process group or any child),
	* and rc is specific child
	* - errno ECHILD (pid does not exist or is not child)
	* - errno EINVAL (invalid flags)
	* - errno EINTR (caller interrupted by signal)
	*
	* @TODO Handle these cases more specifically.
	*/
	signo = SIGCHLD;
	exitcode = 1;
	crm_notice("Wait for child process %d (%s) interrupted: %s",
	child->pid, child->desc, pcmk_strerror(errno));

	} else if (WIFEXITED(status)) {
	exitcode = WEXITSTATUS(status);
	crm_trace("Child process %d (%s) exited with status %d",
	child->pid, child->desc, exitcode);

	} else if (WIFSIGNALED(status)) {
	signo = WTERMSIG(status);
	crm_trace("Child process %d (%s) exited with signal %d (%s)",
	child->pid, child->desc, signo, strsignal(signo));

	#ifdef WCOREDUMP // AIX, SunOS, maybe others
	} else if (WCOREDUMP(status)) {
	core = 1;
	crm_err("Child process %d (%s) dumped core",
	child->pid, child->desc);
	#endif

	} else { // flags must contain WUNTRACED and/or WCONTINUED to reach this
	crm_trace("Child process %d (%s) stopped or continued",
	child->pid, child->desc);
	callback_needed = false;
	}

	if (callback_needed && child->callback) {
	child->callback(child, child->pid, core, signo, exitcode);
	}
	return callback_needed;
	}

	static void
	child_death_dispatch(int signal)
	{
	for (GList *iter = child_list; iter; ) {
	GList *saved = iter;
	mainloop_child_t *child = iter->data;

	iter = iter->next;
	if (child_waitpid(child, WNOHANG)) {
	crm_trace("Removing completed process %d from child list",
	child->pid);
	child_list = g_list_remove_link(child_list, saved);
	g_list_free(saved);
	child_free(child);
	}
	}
	}

	static gboolean
	child_signal_init(gpointer p)
	{
	crm_trace("Installed SIGCHLD handler");
	/* Do NOT use g_child_watch_add() and friends, they rely on pthreads */
	mainloop_add_signal(SIGCHLD, child_death_dispatch);

	/* In case they terminated before the signal handler was installed */
	child_death_dispatch(SIGCHLD);
	return FALSE;
	}

	gboolean
	mainloop_child_kill(pid_t pid)
	{
	GList *iter;
	mainloop_child_t *child = NULL;
	mainloop_child_t *match = NULL;
	/* It is impossible to block SIGKILL, this allows us to
	* call waitpid without WNOHANG flag.*/
	int waitflags = 0, rc = 0;

	for (iter = child_list; iter != NULL && match == NULL; iter = iter->next) {
	child = iter->data;
	if (pid == child->pid) {
	match = child;
	}
	}

	if (match == NULL) {
	return FALSE;
	}

	rc = child_kill_helper(match);
	if(rc == -ESRCH) {
	/* It's gone, but hasn't shown up in waitpid() yet. Wait until we get
	* SIGCHLD and let handler clean it up as normal (so we get the correct
	* return code/status). The blocking alternative would be to call
	* child_waitpid(match, 0).
	*/
	crm_trace("Waiting for signal that child process %d completed",
	match->pid);
	return TRUE;

	} else if(rc != 0) {
	/* If KILL for some other reason set the WNOHANG flag since we
	* can't be certain what happened.
	*/
	waitflags = WNOHANG;
	}

	if (!child_waitpid(match, waitflags)) {
	/* not much we can do if this occurs */
	return FALSE;
	}

	child_list = g_list_remove(child_list, match);
	child_free(match);
	return TRUE;
	}

	/* Create/Log a new tracked process
	* To track a process group, use -pid
	*
	* @TODO Using a non-positive pid (i.e. any child, or process group) would
	* likely not be useful since we will free the child after the first
	* completed process.
	*/
	void
	mainloop_child_add_with_flags(pid_t pid, int timeout, const char desc, void privatedata, enum mainloop_child_flags flags,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	static bool need_init = TRUE;
	mainloop_child_t *child = g_new(mainloop_child_t, 1);

	child->pid = pid;
	child->timerid = 0;
	child->timeout = FALSE;
	child->privatedata = privatedata;
	child->callback = callback;
	child->flags = flags;

	if(desc) {
	child->desc = strdup(desc);
	}

	if (timeout) {
	child->timerid = g_timeout_add(timeout, child_timeout_callback, child);
	}

	child_list = g_list_append(child_list, child);

	if(need_init) {
	need_init = FALSE;
	/* SIGCHLD processing has to be invoked from mainloop.
	* We do not want it to be possible to both add a child pid
	* to mainloop, and have the pid's exit callback invoked within
	* the same callstack. */
	g_timeout_add(1, child_signal_init, NULL);
	}
	}

	void
	mainloop_child_add(pid_t pid, int timeout, const char desc, void privatedata,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback);
	}

	static gboolean
	mainloop_timer_cb(gpointer user_data)
	{
	int id = 0;
	bool repeat = FALSE;
	struct mainloop_timer_s *t = user_data;

	CRM_ASSERT(t != NULL);

	id = t->id;
	t->id = 0; /* Ensure it's unset during callbacks so that
	* mainloop_timer_running() works as expected
	*/

	if(t->cb) {
	crm_trace("Invoking callbacks for timer %s", t->name);
	repeat = t->repeat;
	if(t->cb(t->userdata) == FALSE) {
	crm_trace("Timer %s complete", t->name);
	repeat = FALSE;
	}
	}

	if(repeat) {
	/* Restore if repeating */
	t->id = id;
	}

	return repeat;
	}

	bool
	mainloop_timer_running(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	return TRUE;
	}
	return FALSE;
	}

	void
	mainloop_timer_start(mainloop_timer_t *t)
	{
	mainloop_timer_stop(t);
	if(t && t->period_ms > 0) {
	crm_trace("Starting timer %s", t->name);
	t->id = g_timeout_add(t->period_ms, mainloop_timer_cb, t);
	}
	}

	void
	mainloop_timer_stop(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	crm_trace("Stopping timer %s", t->name);
	g_source_remove(t->id);
	t->id = 0;
	}
	}

	guint
	mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms)
	{
	guint last = 0;

	if(t) {
	last = t->period_ms;
	t->period_ms = period_ms;
	}

	if(t && t->id != 0 && last != t->period_ms) {
	mainloop_timer_start(t);
	}
	return last;
	}

	mainloop_timer_t *
	mainloop_timer_add(const char name, guint period_ms, bool repeat, GSourceFunc cb, void userdata)
	{
	mainloop_timer_t *t = calloc(1, sizeof(mainloop_timer_t));

	if(t) {
	if(name) {
	t->name = crm_strdup_printf("%s-%u-%d", name, period_ms, repeat);
	} else {
	t->name = crm_strdup_printf("%p-%u-%d", t, period_ms, repeat);
	}
	t->id = 0;
	t->period_ms = period_ms;
	t->repeat = repeat;
	t->cb = cb;
	t->userdata = userdata;
	crm_trace("Created timer %s with %p %p", t->name, userdata, t->userdata);
	}
	return t;
	}

	void
	mainloop_timer_del(mainloop_timer_t *t)
	{
	if(t) {
	crm_trace("Destroying timer %s", t->name);
	mainloop_timer_stop(t);
	free(t->name);
	free(t);
	}
	}

	/*
	* Helpers to make sure certain events aren't lost at shutdown
	*/

	static gboolean
	drain_timeout_cb(gpointer user_data)
	{
	bool timeout_popped = (bool) user_data;

	*timeout_popped = TRUE;
	return FALSE;
	}

	/*!
	* \brief Drain some remaining main loop events then quit it
	*
	* \param[in] mloop Main loop to drain and quit
	* \param[in] n Drain up to this many pending events
	*/
	void
	pcmk_quit_main_loop(GMainLoop *mloop, unsigned int n)
	{
	if ((mloop != NULL) && g_main_loop_is_running(mloop)) {
	GMainContext *ctx = g_main_loop_get_context(mloop);

	/* Drain up to n events in case some memory clean-up is pending
	* (helpful to reduce noise in valgrind output).
	*/
	for (int i = 0; (i < n) && g_main_context_pending(ctx); ++i) {
	g_main_context_dispatch(ctx);
	}
	g_main_loop_quit(mloop);
	}
	}

	/*!
	* \brief Process main loop events while a certain condition is met
	*
	* \param[in] mloop Main loop to process
	* \param[in] timer_ms Don't process longer than this amount of time
	* \param[in] check Function that returns TRUE if events should be processed
	*
	* \note This function is intended to be called at shutdown if certain important
	* events should not be missed. The caller would likely quit the main loop
	* or exit after calling this function. The check() function will be
	* passed the remaining timeout in milliseconds.
	*/
	void
	pcmk_drain_main_loop(GMainLoop mloop, guint timer_ms, bool (check)(guint))
	{
	bool timeout_popped = FALSE;
	guint timer = 0;
	GMainContext *ctx = NULL;

	CRM_CHECK(mloop && check, return);

	ctx = g_main_loop_get_context(mloop);
	if (ctx) {
	time_t start_time = time(NULL);

	timer = g_timeout_add(timer_ms, drain_timeout_cb, &timeout_popped);
	while (!timeout_popped
	&& check(timer_ms - (time(NULL) - start_time) * 1000)) {
	g_main_context_iteration(ctx, TRUE);
	}
	}
	if (!timeout_popped && (timer > 0)) {
	g_source_remove(timer);
	}
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/mainloop_compat.h>

	gboolean
	crm_signal(int sig, void (*dispatch) (int sig))
	{
	return crm_signal_handler(sig, dispatch) != SIG_ERR;
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
	index d02aaca84d..75860a13a8 100644
	--- a/lib/services/services_linux.c
	+++ b/lib/services/services_linux.c
	@@ -1,1341 +1,1341 @@
	/*
	* Copyright 2010-2021 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <unistd.h>
	#include <dirent.h>
	#include <grp.h>
	#include <string.h>
	#include <sys/time.h>
	#include <sys/resource.h>

	#include "crm/crm.h"
	#include "crm/common/mainloop.h"
	#include "crm/services.h"
	#include "crm/services_internal.h"

	#include "services_private.h"

	static void close_pipe(int fildes[]);

	/* We have two alternative ways of handling SIGCHLD when synchronously waiting
	* for spawned processes to complete. Both rely on polling a file descriptor to
	* discover SIGCHLD events.
	*
	* If sys/signalfd.h is available (e.g. on Linux), we call signalfd() to
	* generate the file descriptor. Otherwise, we use the "self-pipe trick"
	* (opening a pipe and writing a byte to it when SIGCHLD is received).
	*/
	#ifdef HAVE_SYS_SIGNALFD_H

	// signalfd() implementation

	#include <sys/signalfd.h>

	// Everything needed to manage SIGCHLD handling
	struct sigchld_data_s {
	sigset_t mask; // Signals to block now (including SIGCHLD)
	sigset_t old_mask; // Previous set of blocked signals
	};

	// Initialize SIGCHLD data and prepare for use
	static bool
	sigchld_setup(struct sigchld_data_s *data)
	{
	sigemptyset(&(data->mask));
	sigaddset(&(data->mask), SIGCHLD);

	sigemptyset(&(data->old_mask));

	// Block SIGCHLD (saving previous set of blocked signals to restore later)
	if (sigprocmask(SIG_BLOCK, &(data->mask), &(data->old_mask)) < 0) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=sigprocmask", pcmk_strerror(errno));
	return false;
	}
	return true;
	}

	// Get a file descriptor suitable for polling for SIGCHLD events
	static int
	sigchld_open(struct sigchld_data_s *data)
	{
	int fd;

	CRM_CHECK(data != NULL, return -1);

	fd = signalfd(-1, &(data->mask), SFD_NONBLOCK);
	if (fd < 0) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=signalfd", pcmk_strerror(errno));
	}
	return fd;
	}

	// Close a file descriptor returned by sigchld_open()
	static void
	sigchld_close(int fd)
	{
	if (fd > 0) {
	close(fd);
	}
	}

	// Return true if SIGCHLD was received from polled fd
	static bool
	sigchld_received(int fd)
	{
	struct signalfd_siginfo fdsi;
	ssize_t s;

	if (fd < 0) {
	return false;
	}
	s = read(fd, &fdsi, sizeof(struct signalfd_siginfo));
	if (s != sizeof(struct signalfd_siginfo)) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=read", pcmk_strerror(errno));

	} else if (fdsi.ssi_signo == SIGCHLD) {
	return true;
	}
	return false;
	}

	// Do anything needed after done waiting for SIGCHLD
	static void
	sigchld_cleanup(struct sigchld_data_s *data)
	{
	// Restore the original set of blocked signals
	if ((sigismember(&(data->old_mask), SIGCHLD) == 0)
	&& (sigprocmask(SIG_UNBLOCK, &(data->mask), NULL) < 0)) {
	crm_warn("Could not clean up after child process completion: %s",
	pcmk_strerror(errno));
	}
	}

	#else // HAVE_SYS_SIGNALFD_H not defined

	// Self-pipe implementation (see above for function descriptions)

	struct sigchld_data_s {
	int pipe_fd[2]; // Pipe file descriptors
	struct sigaction sa; // Signal handling info (with SIGCHLD)
	struct sigaction old_sa; // Previous signal handling info
	};

	// We need a global to use in the signal handler
	volatile struct sigchld_data_s *last_sigchld_data = NULL;

	static void
	sigchld_handler()
	{
	// We received a SIGCHLD, so trigger pipe polling
	if ((last_sigchld_data != NULL)
	&& (last_sigchld_data->pipe_fd[1] >= 0)
	&& (write(last_sigchld_data->pipe_fd[1], "", 1) == -1)) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=write", pcmk_strerror(errno));
	}
	}

	static bool
	sigchld_setup(struct sigchld_data_s *data)
	{
	int rc;

	data->pipe_fd[0] = data->pipe_fd[1] = -1;

	if (pipe(data->pipe_fd) == -1) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=pipe", pcmk_strerror(errno));
	return false;
	}

	rc = pcmk__set_nonblocking(data->pipe_fd[0]);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set pipe input non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}
	rc = pcmk__set_nonblocking(data->pipe_fd[1]);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set pipe output non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}

	// Set SIGCHLD handler
	data->sa.sa_handler = sigchld_handler;
	data->sa.sa_flags = 0;
	sigemptyset(&(data->sa.sa_mask));
	if (sigaction(SIGCHLD, &(data->sa), &(data->old_sa)) < 0) {
	crm_err("Wait for child process completion failed: %s "
	CRM_XS " source=sigaction", pcmk_strerror(errno));
	}

	// Remember data for use in signal handler
	last_sigchld_data = data;
	return true;
	}

	static int
	sigchld_open(struct sigchld_data_s *data)
	{
	CRM_CHECK(data != NULL, return -1);
	return data->pipe_fd[0];
	}

	static void
	sigchld_close(int fd)
	{
	// Pipe will be closed in sigchld_cleanup()
	return;
	}

	static bool
	sigchld_received(int fd)
	{
	char ch;

	if (fd < 0) {
	return false;
	}

	// Clear out the self-pipe
	while (read(fd, &ch, 1) == 1) /omit/;
	return true;
	}

	static void
	sigchld_cleanup(struct sigchld_data_s *data)
	{
	// Restore the previous SIGCHLD handler
	if (sigaction(SIGCHLD, &(data->old_sa), NULL) < 0) {
	crm_warn("Could not clean up after child process completion: %s",
	pcmk_strerror(errno));
	}

	close_pipe(data->pipe_fd);
	}

	#endif

	/*!
	* \internal
	* \brief Close the two file descriptors of a pipe
	*
	* \param[in] fildes Array of file descriptors opened by pipe()
	*/
	static void
	close_pipe(int fildes[])
	{
	if (fildes[0] >= 0) {
	close(fildes[0]);
	fildes[0] = -1;
	}
	if (fildes[1] >= 0) {
	close(fildes[1]);
	fildes[1] = -1;
	}
	}

	static gboolean
	svc_read_output(int fd, svc_action_t * op, bool is_stderr)
	{
	char *data = NULL;
	int rc = 0, len = 0;
	char buf[500];
	static const size_t buf_read_len = sizeof(buf) - 1;


	if (fd < 0) {
	crm_trace("No fd for %s", op->id);
	return FALSE;
	}

	if (is_stderr && op->stderr_data) {
	len = strlen(op->stderr_data);
	data = op->stderr_data;
	crm_trace("Reading %s stderr into offset %d", op->id, len);

	} else if (is_stderr == FALSE && op->stdout_data) {
	len = strlen(op->stdout_data);
	data = op->stdout_data;
	crm_trace("Reading %s stdout into offset %d", op->id, len);

	} else {
	crm_trace("Reading %s %s into offset %d", op->id, is_stderr?"stderr":"stdout", len);
	}

	do {
	rc = read(fd, buf, buf_read_len);
	if (rc > 0) {
	buf[rc] = 0;
	crm_trace("Got %d chars: %.80s", rc, buf);
	data = pcmk__realloc(data, len + rc + 1);
	len += sprintf(data + len, "%s", buf);

	} else if (errno != EINTR) {
	/* error or EOF
	* Cleanup happens in pipe_done()
	*/
	rc = FALSE;
	break;
	}

	} while (rc == buf_read_len \|\| rc < 0);

	if (is_stderr) {
	op->stderr_data = data;
	} else {
	op->stdout_data = data;
	}

	return rc;
	}

	static int
	dispatch_stdout(gpointer userdata)
	{
	svc_action_t op = (svc_action_t ) userdata;

	return svc_read_output(op->opaque->stdout_fd, op, FALSE);
	}

	static int
	dispatch_stderr(gpointer userdata)
	{
	svc_action_t op = (svc_action_t ) userdata;

	return svc_read_output(op->opaque->stderr_fd, op, TRUE);
	}

	static void
	pipe_out_done(gpointer user_data)
	{
	svc_action_t op = (svc_action_t ) user_data;

	crm_trace("%p", op);

	op->opaque->stdout_gsource = NULL;
	if (op->opaque->stdout_fd > STDOUT_FILENO) {
	close(op->opaque->stdout_fd);
	}
	op->opaque->stdout_fd = -1;
	}

	static void
	pipe_err_done(gpointer user_data)
	{
	svc_action_t op = (svc_action_t ) user_data;

	op->opaque->stderr_gsource = NULL;
	if (op->opaque->stderr_fd > STDERR_FILENO) {
	close(op->opaque->stderr_fd);
	}
	op->opaque->stderr_fd = -1;
	}

	static struct mainloop_fd_callbacks stdout_callbacks = {
	.dispatch = dispatch_stdout,
	.destroy = pipe_out_done,
	};

	static struct mainloop_fd_callbacks stderr_callbacks = {
	.dispatch = dispatch_stderr,
	.destroy = pipe_err_done,
	};

	static void
	set_ocf_env(const char key, const char value, gpointer user_data)
	{
	if (setenv(key, value, 1) != 0) {
	crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
	}
	}

	static void
	set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
	{
	char buffer[500];

	snprintf(buffer, sizeof(buffer), strcmp(key, "OCF_CHECK_LEVEL") != 0 ? "OCF_RESKEY_%s" : "%s", (char *)key);
	set_ocf_env(buffer, value, user_data);
	}

	static void
	set_alert_env(gpointer key, gpointer value, gpointer user_data)
	{
	int rc;

	if (value != NULL) {
	rc = setenv(key, value, 1);
	} else {
	rc = unsetenv(key);
	}

	if (rc < 0) {
	crm_perror(LOG_ERR, "setenv %s=%s",
	(char)key, (value? (char)value : ""));
	} else {
	crm_trace("setenv %s=%s", (char)key, (value? (char)value : ""));
	}
	}

	/*!
	* \internal
	* \brief Add environment variables suitable for an action
	*
	* \param[in] op Action to use
	*/
	static void
	add_action_env_vars(const svc_action_t *op)
	{
	void (*env_setter)(gpointer, gpointer, gpointer) = NULL;
	if (op->agent == NULL) {
	env_setter = set_alert_env; /* we deal with alert handler */

	} else if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
	env_setter = set_ocf_env_with_prefix;
	}

	if (env_setter != NULL && op->params != NULL) {
	g_hash_table_foreach(op->params, env_setter, NULL);
	}

	if (env_setter == NULL \|\| env_setter == set_alert_env) {
	return;
	}

	set_ocf_env("OCF_RA_VERSION_MAJOR", PCMK_OCF_MAJOR_VERSION, NULL);
	set_ocf_env("OCF_RA_VERSION_MINOR", PCMK_OCF_MINOR_VERSION, NULL);
	set_ocf_env("OCF_ROOT", OCF_ROOT_DIR, NULL);
	set_ocf_env("OCF_EXIT_REASON_PREFIX", PCMK_OCF_REASON_PREFIX, NULL);

	if (op->rsc) {
	set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
	}

	if (op->agent != NULL) {
	set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
	}

	/* Notes: this is not added to specification yet. Sept 10,2004 */
	if (op->provider != NULL) {
	set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
	}
	}

	static void
	pipe_in_single_parameter(gpointer key, gpointer value, gpointer user_data)
	{
	svc_action_t *op = user_data;
	char buffer = crm_strdup_printf("%s=%s\n", (char )key, (char *) value);
	int ret, total = 0, len = strlen(buffer);

	do {
	errno = 0;
	ret = write(op->opaque->stdin_fd, buffer + total, len - total);
	if (ret > 0) {
	total += ret;
	}

	} while ((errno == EINTR) && (total < len));
	free(buffer);
	}

	/*!
	* \internal
	* \brief Pipe parameters in via stdin for action
	*
	* \param[in] op Action to use
	*/
	static void
	pipe_in_action_stdin_parameters(const svc_action_t *op)
	{
	crm_debug("sending args");
	if (op->params) {
	g_hash_table_foreach(op->params, pipe_in_single_parameter, (gpointer) op);
	}
	}

	gboolean
	recurring_action_timer(gpointer data)
	{
	svc_action_t *op = data;

	crm_debug("Scheduling another invocation of %s", op->id);

	/* Clean out the old result */
	free(op->stdout_data);
	op->stdout_data = NULL;
	free(op->stderr_data);
	op->stderr_data = NULL;
	op->opaque->repeat_timer = 0;

	services_action_async(op, NULL);
	return FALSE;
	}

	/*!
	* \internal
	* \brief Finalize handling of an asynchronous operation
	*
	* Given a completed asynchronous operation, cancel or reschedule it as
	* appropriate if recurring, call its callback if registered, stop tracking it,
	* and clean it up.
	*
	* \param[in,out] op Operation to finalize
	*
	* \return Standard Pacemaker return code
	* \retval EINVAL Caller supplied NULL or invalid \p op
	* \retval EBUSY Uncanceled recurring action has only been cleaned up
	* \retval pcmk_rc_ok Action has been freed
	*
	* \note If the return value is not pcmk_rc_ok, the caller is responsible for
	* freeing the action.
	*/
	int
	services__finalize_async_op(svc_action_t *op)
	{
	CRM_CHECK((op != NULL) && !(op->synchronous), return EINVAL);

	if (op->interval_ms != 0) {
	// Recurring operations must be either cancelled or rescheduled
	if (op->cancel) {
	services__set_cancelled(op);
	cancel_recurring_action(op);
	} else {
	op->opaque->repeat_timer = g_timeout_add(op->interval_ms,
	recurring_action_timer,
	(void *) op);
	}
	}

	if (op->opaque->callback != NULL) {
	op->opaque->callback(op);
	}

	// Stop tracking the operation (as in-flight or blocked)
	op->pid = 0;
	services_untrack_op(op);

	if ((op->interval_ms != 0) && !(op->cancel)) {
	// Do not free recurring actions (they will get freed when cancelled)
	services_action_cleanup(op);
	return EBUSY;
	}

	services_action_free(op);
	return pcmk_rc_ok;
	}

	static void
	close_op_input(svc_action_t *op)
	{
	if (op->opaque->stdin_fd >= 0) {
	close(op->opaque->stdin_fd);
	}
	}

	static void
	finish_op_output(svc_action_t *op, bool is_stderr)
	{
	mainloop_io_t **source;
	int fd;

	if (is_stderr) {
	source = &(op->opaque->stderr_gsource);
	fd = op->opaque->stderr_fd;
	} else {
	source = &(op->opaque->stdout_gsource);
	fd = op->opaque->stdout_fd;
	}

	if (op->synchronous \|\| *source) {
	crm_trace("Finish reading %s[%d] %s",
	op->id, op->pid, (is_stderr? "stdout" : "stderr"));
	svc_read_output(fd, op, is_stderr);
	if (op->synchronous) {
	close(fd);
	} else {
	mainloop_del_fd(*source);
	*source = NULL;
	}
	}
	}

	// Log an operation's stdout and stderr
	static void
	log_op_output(svc_action_t *op)
	{
	char *prefix = crm_strdup_printf("%s[%d] error output", op->id, op->pid);

	crm_log_output(LOG_NOTICE, prefix, op->stderr_data);
	strcpy(prefix + strlen(prefix) - strlen("error output"), "output");
	crm_log_output(LOG_DEBUG, prefix, op->stdout_data);
	free(prefix);
	}

	// Truncate exit reasons at this many characters
	#define EXIT_REASON_MAX_LEN 128

	static void
	parse_exit_reason_from_stderr(svc_action_t *op)
	{
	const char *reason_start = NULL;
	const char *reason_end = NULL;
	const int prefix_len = strlen(PCMK_OCF_REASON_PREFIX);

	if ((op->stderr_data == NULL) \|\|
	// Only OCF agents have exit reasons in stderr
	!pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_none)) {
	return;
	}

	// Find the last occurrence of the magic string indicating an exit reason
	for (const char *cur = strstr(op->stderr_data, PCMK_OCF_REASON_PREFIX);
	cur != NULL; cur = strstr(cur, PCMK_OCF_REASON_PREFIX)) {

	cur += prefix_len; // Skip over magic string
	reason_start = cur;
	}

	if ((reason_start == NULL) \|\| (reason_start[0] == '\n')
	\|\| (reason_start[0] == '\0')) {
	return; // No or empty exit reason
	}

	// Exit reason goes to end of line (or end of output)
	reason_end = strchr(reason_start, '\n');
	if (reason_end == NULL) {
	reason_end = reason_start + strlen(reason_start);
	}

	// Limit size of exit reason to something reasonable
	if (reason_end > (reason_start + EXIT_REASON_MAX_LEN)) {
	reason_end = reason_start + EXIT_REASON_MAX_LEN;
	}

	free(op->opaque->exit_reason);
	op->opaque->exit_reason = strndup(reason_start, reason_end - reason_start);
	}

	/*!
	* \internal
	* \brief Process the completion of an asynchronous child process
	*
	* \param[in] p Child process that completed
	* \param[in] pid Process ID of child
	* \param[in] core (unused)
	* \param[in] signo Signal that interrupted child, if any
	* \param[in] exitcode Exit status of child process
	*/
	static void
	async_action_complete(mainloop_child_t *p, pid_t pid, int core, int signo,
	int exitcode)
	{
	svc_action_t *op = mainloop_child_userdata(p);

	mainloop_clear_child_userdata(p);
	CRM_CHECK(op->pid == pid,
	services__set_result(op, services__generic_error(op),
	PCMK_EXEC_ERROR, "Bug in mainloop handling");
	return);

	/* Depending on the priority the mainloop gives the stdout and stderr
	* file descriptors, this function could be called before everything has
	* been read from them, so force a final read now.
	*/
	finish_op_output(op, true);
	finish_op_output(op, false);

	close_op_input(op);

	if (signo == 0) {
	crm_debug("%s[%d] exited with status %d", op->id, op->pid, exitcode);
	services__set_result(op, exitcode, PCMK_EXEC_DONE, NULL);
	log_op_output(op);
	parse_exit_reason_from_stderr(op);

	} else if (mainloop_child_timeout(p)) {
	crm_warn("%s[%d] timed out after %dms", op->id, op->pid, op->timeout);
	services__set_result(op, services__generic_error(op), PCMK_EXEC_TIMEOUT,
	"Process did not exit within specified timeout");

	} else if (op->cancel) {
	/* If an in-flight recurring operation was killed because it was
	* cancelled, don't treat that as a failure.
	*/
	- crm_info("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
	- op->id, op->pid, strsignal(signo), signo);
	+ crm_info("%s[%d] terminated with signal %d (%s)",
	+ op->id, op->pid, signo, strsignal(signo));
	services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_CANCELLED, NULL);

	} else {
	- crm_warn("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
	- op->id, op->pid, strsignal(signo), signo);
	+ crm_warn("%s[%d] terminated with signal %d (%s)",
	+ op->id, op->pid, signo, strsignal(signo));
	services__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Process interrupted by signal");
	}

	services__finalize_async_op(op);
	}

	/*!
	* \internal
	* \brief Return agent standard's exit status for "generic error"
	*
	* When returning an internal error for an action, a value that is appropriate
	* to the action's agent standard must be used. This function returns a value
	* appropriate for errors in general.
	*
	* \param[in] op Action that error is for
	*
	* \return Exit status appropriate to agent standard
	* \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
	*/
	int
	services__generic_error(svc_action_t *op)
	{
	if ((op == NULL) \|\| (op->standard == NULL)) {
	return PCMK_OCF_UNKNOWN_ERROR;
	}

	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
	&& pcmk__str_eq(op->action, "status", pcmk__str_casei)) {

	return PCMK_LSB_STATUS_UNKNOWN;
	}

	#if SUPPORT_NAGIOS
	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
	return NAGIOS_STATE_UNKNOWN;
	}
	#endif

	return PCMK_OCF_UNKNOWN_ERROR;
	}

	/*!
	* \internal
	* \brief Return agent standard's exit status for "not installed"
	*
	* When returning an internal error for an action, a value that is appropriate
	* to the action's agent standard must be used. This function returns a value
	* appropriate for "not installed" errors.
	*
	* \param[in] op Action that error is for
	*
	* \return Exit status appropriate to agent standard
	* \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
	*/
	int
	services__not_installed_error(svc_action_t *op)
	{
	if ((op == NULL) \|\| (op->standard == NULL)) {
	return PCMK_OCF_UNKNOWN_ERROR;
	}

	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
	&& pcmk__str_eq(op->action, "status", pcmk__str_casei)) {

	return PCMK_LSB_STATUS_NOT_INSTALLED;
	}

	#if SUPPORT_NAGIOS
	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
	return NAGIOS_STATE_UNKNOWN;
	}
	#endif

	return PCMK_OCF_NOT_INSTALLED;
	}

	/*!
	* \internal
	* \brief Return agent standard's exit status for "insufficient privileges"
	*
	* When returning an internal error for an action, a value that is appropriate
	* to the action's agent standard must be used. This function returns a value
	* appropriate for "insufficient privileges" errors.
	*
	* \param[in] op Action that error is for
	*
	* \return Exit status appropriate to agent standard
	* \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
	*/
	int
	services__authorization_error(svc_action_t *op)
	{
	if ((op == NULL) \|\| (op->standard == NULL)) {
	return PCMK_OCF_UNKNOWN_ERROR;
	}

	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
	&& pcmk__str_eq(op->action, "status", pcmk__str_casei)) {

	return PCMK_LSB_STATUS_INSUFFICIENT_PRIV;
	}

	#if SUPPORT_NAGIOS
	if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
	return NAGIOS_INSUFFICIENT_PRIV;
	}
	#endif

	return PCMK_OCF_INSUFFICIENT_PRIV;
	}

	/*!
	* \internal
	* \brief Set operation rc and status per errno from stat(), fork() or execvp()
	*
	* \param[in,out] op Operation to set rc and status for
	* \param[in] error Value of errno after system call
	*
	* \return void
	*/
	void
	services__handle_exec_error(svc_action_t * op, int error)
	{
	switch (error) { /* see execve(2), stat(2) and fork(2) */
	case ENOENT: /* No such file or directory */
	case EISDIR: /* Is a directory */
	case ENOTDIR: /* Path component is not a directory */
	case EINVAL: /* Invalid executable format */
	case ENOEXEC: /* Invalid executable format */
	services__set_result(op, services__not_installed_error(op),
	PCMK_EXEC_NOT_INSTALLED, pcmk_rc_str(error));
	break;
	case EACCES: /* permission denied (various errors) */
	case EPERM: /* permission denied (various errors) */
	services__set_result(op, services__authorization_error(op),
	PCMK_EXEC_ERROR, pcmk_rc_str(error));
	break;
	default:
	services__set_result(op, services__generic_error(op),
	PCMK_EXEC_ERROR, pcmk_rc_str(error));
	}
	}

	static void
	action_launch_child(svc_action_t *op)
	{
	/* SIGPIPE is ignored (which is different from signal blocking) by the gnutls library.
	* Depending on the libqb version in use, libqb may set SIGPIPE to be ignored as well.
	* We do not want this to be inherited by the child process. By resetting this the signal
	* to the default behavior, we avoid some potential odd problems that occur during OCF
	* scripts when SIGPIPE is ignored by the environment. */
	signal(SIGPIPE, SIG_DFL);

	#if defined(HAVE_SCHED_SETSCHEDULER)
	if (sched_getscheduler(0) != SCHED_OTHER) {
	struct sched_param sp;

	memset(&sp, 0, sizeof(sp));
	sp.sched_priority = 0;

	if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) {
	crm_perror(LOG_ERR, "Could not reset scheduling policy to SCHED_OTHER for %s", op->id);
	}
	}
	#endif
	if (setpriority(PRIO_PROCESS, 0, 0) == -1) {
	crm_perror(LOG_ERR, "Could not reset process priority to 0 for %s", op->id);
	}

	/* Man: The call setpgrp() is equivalent to setpgid(0,0)
	* _and_ compiles on BSD variants too
	* need to investigate if it works the same too.
	*/
	setpgid(0, 0);

	pcmk__close_fds_in_child(false);

	#if SUPPORT_CIBSECRETS
	if (pcmk__substitute_secrets(op->rsc, op->params) != pcmk_rc_ok) {
	/* replacing secrets failed! */
	if (pcmk__str_eq(op->action, "stop", pcmk__str_casei)) {
	/* don't fail on stop! */
	crm_info("proceeding with the stop operation for %s", op->rsc);

	} else {
	crm_err("failed to get secrets for %s, "
	"considering resource not configured", op->rsc);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}
	}
	#endif

	add_action_env_vars(op);

	/* Become the desired user */
	if (op->opaque->uid && (geteuid() == 0)) {

	// If requested, set effective group
	if (op->opaque->gid && (setgid(op->opaque->gid) < 0)) {
	crm_perror(LOG_ERR, "Could not set child group to %d", op->opaque->gid);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}

	// Erase supplementary group list
	// (We could do initgroups() if we kept a copy of the username)
	if (setgroups(0, NULL) < 0) {
	crm_perror(LOG_ERR, "Could not set child groups");
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}

	// Set effective user
	if (setuid(op->opaque->uid) < 0) {
	crm_perror(LOG_ERR, "setting user to %d", op->opaque->uid);
	_exit(PCMK_OCF_NOT_CONFIGURED);
	}
	}

	/* execute the RA */
	execvp(op->opaque->exec, op->opaque->args);

	/* Most cases should have been already handled by stat() */
	services__handle_exec_error(op, errno);

	_exit(op->rc);
	}

	/*!
	* \internal
	* \brief Wait for synchronous action to complete, and set its result
	*
	* \param[in] op Action to wait for
	* \param[in] data Child signal data
	*/
	static void
	wait_for_sync_result(svc_action_t op, struct sigchld_data_s data)
	{
	int status = 0;
	int timeout = op->timeout;
	time_t start = time(NULL);
	struct pollfd fds[3];
	int wait_rc = 0;
	const char *wait_reason = NULL;

	fds[0].fd = op->opaque->stdout_fd;
	fds[0].events = POLLIN;
	fds[0].revents = 0;

	fds[1].fd = op->opaque->stderr_fd;
	fds[1].events = POLLIN;
	fds[1].revents = 0;

	fds[2].fd = sigchld_open(data);
	fds[2].events = POLLIN;
	fds[2].revents = 0;

	crm_trace("Waiting for %s[%d]", op->id, op->pid);
	do {
	int poll_rc = poll(fds, 3, timeout);

	wait_reason = NULL;

	if (poll_rc > 0) {
	if (fds[0].revents & POLLIN) {
	svc_read_output(op->opaque->stdout_fd, op, FALSE);
	}

	if (fds[1].revents & POLLIN) {
	svc_read_output(op->opaque->stderr_fd, op, TRUE);
	}

	if ((fds[2].revents & POLLIN) && sigchld_received(fds[2].fd)) {
	wait_rc = waitpid(op->pid, &status, WNOHANG);

	if ((wait_rc > 0) \|\| ((wait_rc < 0) && (errno == ECHILD))) {
	// Child process exited or doesn't exist
	break;

	} else if (wait_rc < 0) {
	wait_reason = pcmk_rc_str(errno);
	crm_warn("Wait for completion of %s[%d] failed: %s "
	CRM_XS " source=waitpid",
	op->id, op->pid, wait_reason);
	wait_rc = 0; // Act as if process is still running
	}
	}

	} else if (poll_rc == 0) {
	// Poll timed out with no descriptors ready
	timeout = 0;
	break;

	} else if ((poll_rc < 0) && (errno != EINTR)) {
	wait_reason = pcmk_rc_str(errno);
	crm_err("Wait for completion of %s[%d] failed: %s "
	CRM_XS " source=poll", op->id, op->pid, wait_reason);
	break;
	}

	timeout = op->timeout - (time(NULL) - start) * 1000;

	} while ((op->timeout < 0 \|\| timeout > 0));

	crm_trace("Stopped waiting for %s[%d]", op->id, op->pid);
	finish_op_output(op, true);
	finish_op_output(op, false);
	close_op_input(op);
	sigchld_close(fds[2].fd);

	if (wait_rc <= 0) {

	if ((op->timeout > 0) && (timeout <= 0)) {
	services__set_result(op, services__generic_error(op),
	PCMK_EXEC_TIMEOUT,
	"Process did not exit within specified timeout");
	crm_warn("%s[%d] timed out after %dms",
	op->id, op->pid, op->timeout);

	} else {
	services__set_result(op, services__generic_error(op),
	PCMK_EXEC_ERROR, wait_reason);
	}

	/* If only child hasn't been successfully waited for, yet.
	This is to limit killing wrong target a bit more. */
	if ((wait_rc == 0) && (waitpid(op->pid, &status, WNOHANG) == 0)) {
	if (kill(op->pid, SIGKILL)) {
	crm_warn("Could not kill rogue child %s[%d]: %s",
	op->id, op->pid, pcmk_strerror(errno));
	}
	/* Safe to skip WNOHANG here as we sent non-ignorable signal. */
	while ((waitpid(op->pid, &status, 0) == (pid_t) -1)
	&& (errno == EINTR)) {
	/* keep waiting */;
	}
	}

	} else if (WIFEXITED(status)) {
	services__set_result(op, WEXITSTATUS(status), PCMK_EXEC_DONE, NULL);
	parse_exit_reason_from_stderr(op);
	crm_info("%s[%d] exited with status %d", op->id, op->pid, op->rc);

	} else if (WIFSIGNALED(status)) {
	int signo = WTERMSIG(status);

	services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
	"Process interrupted by signal");
	- crm_err("%s[%d] terminated with signal: %s " CRM_XS " (%d)",
	- op->id, op->pid, strsignal(signo), signo);
	+ crm_err("%s[%d] terminated with signal %d (%s)",
	+ op->id, op->pid, signo, strsignal(signo));

	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	crm_warn("%s[%d] dumped core", op->id, op->pid);
	}
	#endif

	} else {
	// Shouldn't be possible to get here
	services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
	"Unable to wait for child to complete");
	}
	}

	/*!
	* \internal
	* \brief Execute an action whose standard uses executable files
	*
	* \param[in] op Action to execute
	*
	* \return Standard Pacemaker return value
	* \retval EBUSY Recurring operation could not be initiated
	* \retval pcmk_rc_error Synchronous action failed
	* \retval pcmk_rc_ok Synchronous action succeeded, or asynchronous action
	* should not be freed (because it already was or is
	* pending)
	*
	* \note If the return value for an asynchronous action is not pcmk_rc_ok, the
	* caller is responsible for freeing the action.
	*/
	int
	services__execute_file(svc_action_t *op)
	{
	int stdout_fd[2];
	int stderr_fd[2];
	int stdin_fd[2] = {-1, -1};
	int rc;
	struct stat st;
	struct sigchld_data_s data;

	// Catch common failure conditions early
	if (stat(op->opaque->exec, &st) != 0) {
	rc = errno;
	crm_warn("Cannot execute '%s': %s " CRM_XS " stat rc=%d",
	op->opaque->exec, pcmk_strerror(rc), rc);
	services__handle_exec_error(op, rc);
	goto done;
	}

	if (pipe(stdout_fd) < 0) {
	rc = errno;
	crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stdout) rc=%d",
	op->opaque->exec, pcmk_strerror(rc), rc);
	services__handle_exec_error(op, rc);
	goto done;
	}

	if (pipe(stderr_fd) < 0) {
	rc = errno;

	close_pipe(stdout_fd);

	crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stderr) rc=%d",
	op->opaque->exec, pcmk_strerror(rc), rc);
	services__handle_exec_error(op, rc);
	goto done;
	}

	if (pcmk_is_set(pcmk_get_ra_caps(op->standard), pcmk_ra_cap_stdin)) {
	if (pipe(stdin_fd) < 0) {
	rc = errno;

	close_pipe(stdout_fd);
	close_pipe(stderr_fd);

	crm_err("Cannot execute '%s': %s " CRM_XS " pipe(stdin) rc=%d",
	op->opaque->exec, pcmk_strerror(rc), rc);
	services__handle_exec_error(op, rc);
	goto done;
	}
	}

	if (op->synchronous && !sigchld_setup(&data)) {
	close_pipe(stdin_fd);
	close_pipe(stdout_fd);
	close_pipe(stderr_fd);
	sigchld_cleanup(&data);
	services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
	"Could not manage signals for child process");
	goto done;
	}

	op->pid = fork();
	switch (op->pid) {
	case -1:
	rc = errno;
	close_pipe(stdin_fd);
	close_pipe(stdout_fd);
	close_pipe(stderr_fd);

	crm_err("Cannot execute '%s': %s " CRM_XS " fork rc=%d",
	op->opaque->exec, pcmk_strerror(rc), rc);
	services__handle_exec_error(op, rc);
	if (op->synchronous) {
	sigchld_cleanup(&data);
	}
	goto done;
	break;

	case 0: /* Child */
	close(stdout_fd[0]);
	close(stderr_fd[0]);
	if (stdin_fd[1] >= 0) {
	close(stdin_fd[1]);
	}
	if (STDOUT_FILENO != stdout_fd[1]) {
	if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
	crm_warn("Can't redirect output from '%s': %s "
	CRM_XS " errno=%d",
	op->opaque->exec, pcmk_strerror(errno), errno);
	}
	close(stdout_fd[1]);
	}
	if (STDERR_FILENO != stderr_fd[1]) {
	if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
	crm_warn("Can't redirect error output from '%s': %s "
	CRM_XS " errno=%d",
	op->opaque->exec, pcmk_strerror(errno), errno);
	}
	close(stderr_fd[1]);
	}
	if ((stdin_fd[0] >= 0) &&
	(STDIN_FILENO != stdin_fd[0])) {
	if (dup2(stdin_fd[0], STDIN_FILENO) != STDIN_FILENO) {
	crm_warn("Can't redirect input to '%s': %s "
	CRM_XS " errno=%d",
	op->opaque->exec, pcmk_strerror(errno), errno);
	}
	close(stdin_fd[0]);
	}

	if (op->synchronous) {
	sigchld_cleanup(&data);
	}

	action_launch_child(op);
	CRM_ASSERT(0); /* action_launch_child is effectively noreturn */
	}

	/* Only the parent reaches here */
	close(stdout_fd[1]);
	close(stderr_fd[1]);
	if (stdin_fd[0] >= 0) {
	close(stdin_fd[0]);
	}

	op->opaque->stdout_fd = stdout_fd[0];
	rc = pcmk__set_nonblocking(op->opaque->stdout_fd);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set '%s' output non-blocking: %s "
	CRM_XS " rc=%d",
	op->opaque->exec, pcmk_rc_str(rc), rc);
	}

	op->opaque->stderr_fd = stderr_fd[0];
	rc = pcmk__set_nonblocking(op->opaque->stderr_fd);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set '%s' error output non-blocking: %s "
	CRM_XS " rc=%d",
	op->opaque->exec, pcmk_rc_str(rc), rc);
	}

	op->opaque->stdin_fd = stdin_fd[1];
	if (op->opaque->stdin_fd >= 0) {
	// using buffer behind non-blocking-fd here - that could be improved
	// as long as no other standard uses stdin_fd assume stonith
	rc = pcmk__set_nonblocking(op->opaque->stdin_fd);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set '%s' input non-blocking: %s "
	CRM_XS " fd=%d,rc=%d", op->opaque->exec,
	pcmk_rc_str(rc), op->opaque->stdin_fd, rc);
	}
	pipe_in_action_stdin_parameters(op);
	// as long as we are handling parameters directly in here just close
	close(op->opaque->stdin_fd);
	op->opaque->stdin_fd = -1;
	}

	// after fds are setup properly and before we plug anything into mainloop
	if (op->opaque->fork_callback) {
	op->opaque->fork_callback(op);
	}

	if (op->synchronous) {
	wait_for_sync_result(op, &data);
	sigchld_cleanup(&data);
	goto done;
	}

	crm_trace("Waiting async for '%s'[%d]", op->opaque->exec, op->pid);
	mainloop_child_add_with_flags(op->pid, op->timeout, op->id, op,
	pcmk_is_set(op->flags, SVC_ACTION_LEAVE_GROUP)? mainloop_leave_pid_group : 0,
	async_action_complete);

	op->opaque->stdout_gsource = mainloop_add_fd(op->id,
	G_PRIORITY_LOW,
	op->opaque->stdout_fd, op,
	&stdout_callbacks);
	op->opaque->stderr_gsource = mainloop_add_fd(op->id,
	G_PRIORITY_LOW,
	op->opaque->stderr_fd, op,
	&stderr_callbacks);
	services_add_inflight_op(op);
	return pcmk_rc_ok;

	done:
	if (op->synchronous) {
	return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
	} else {
	return services__finalize_async_op(op);
	}
	}

	GList *
	services_os_get_single_directory_list(const char *root, gboolean files, gboolean executable)
	{
	GList *list = NULL;
	struct dirent **namelist;
	int entries = 0, lpc = 0;
	char buffer[PATH_MAX];

	entries = scandir(root, &namelist, NULL, alphasort);
	if (entries <= 0) {
	return list;
	}

	for (lpc = 0; lpc < entries; lpc++) {
	struct stat sb;

	if ('.' == namelist[lpc]->d_name[0]) {
	free(namelist[lpc]);
	continue;
	}

	snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);

	if (stat(buffer, &sb)) {
	continue;
	}

	if (S_ISDIR(sb.st_mode)) {
	if (files) {
	free(namelist[lpc]);
	continue;
	}

	} else if (S_ISREG(sb.st_mode)) {
	if (files == FALSE) {
	free(namelist[lpc]);
	continue;

	} else if (executable
	&& (sb.st_mode & S_IXUSR) == 0
	&& (sb.st_mode & S_IXGRP) == 0 && (sb.st_mode & S_IXOTH) == 0) {
	free(namelist[lpc]);
	continue;
	}
	}

	list = g_list_append(list, strdup(namelist[lpc]->d_name));

	free(namelist[lpc]);
	}

	free(namelist);
	return list;
	}

	GList *
	services_os_get_directory_list(const char *root, gboolean files, gboolean executable)
	{
	GList *result = NULL;
	char *dirs = strdup(root);
	char *dir = NULL;

	if (pcmk__str_empty(dirs)) {
	free(dirs);
	return result;
	}

	for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
	GList *tmp = services_os_get_single_directory_list(dir, files, executable);

	if (tmp) {
	result = g_list_concat(result, tmp);
	}
	}

	free(dirs);

	return result;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 4:24 PM (16 h, 46 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018889
Default Alt Text: (129 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions