No OneTemporary
Actions

Size

425 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cib/main.c b/cib/main.c
	index 8cd1e222fc..68c53b425a 100644
	--- a/cib/main.c
	+++ b/cib/main.c
	@@ -1,598 +1,598 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>

	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include <cibio.h>
	#include <callbacks.h>
	#include <pwd.h>

	#if HAVE_LIBXML2
	# include <libxml/parser.h>
	#endif

	#ifdef HAVE_GETOPT_H
	# include <getopt.h>
	#endif

	#if HAVE_BZLIB_H
	# include <bzlib.h>
	#endif

	extern int init_remote_listener(int port, gboolean encrypted);
	extern gboolean stand_alone;

	gboolean cib_shutdown_flag = FALSE;
	int cib_status = pcmk_ok;

	#if SUPPORT_HEARTBEAT
	oc_ev_t *cib_ev_token;
	ll_cluster_t *hb_conn = NULL;
	extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int);
	gboolean cib_register_ha(ll_cluster_t * hb_cluster, const char *client_name);
	#endif

	extern void terminate_cib(const char *caller, gboolean fast);

	GMainLoop *mainloop = NULL;
	const char *cib_root = CRM_CONFIG_DIR;
	char *cib_our_uname = NULL;
	gboolean preserve_status = FALSE;
	gboolean cib_writes_enabled = TRUE;
	int remote_fd = 0;
	int remote_tls_fd = 0;

	void usage(const char *cmd, int exit_status);
	int cib_init(void);
	void cib_shutdown(int nsig);
	gboolean startCib(const char *filename);
	extern int write_cib_contents(gpointer p);

	GHashTable *client_list = NULL;
	GHashTable *config_hash = NULL;
	GHashTable *local_notify_queue = NULL;

	char *channel1 = NULL;
	char *channel2 = NULL;
	char *channel3 = NULL;
	char *channel4 = NULL;
	char *channel5 = NULL;

	#define OPTARGS "maswr:V?"
	void cib_cleanup(void);

	static void
	cib_enable_writes(int nsig)
	{
	crm_info("(Re)enabling disk writes");
	cib_writes_enabled = TRUE;
	}

	static void
	log_cib_client(gpointer key, gpointer value, gpointer user_data)
	{
	cib_client_t *a_client = value;

	crm_info("Client %s", crm_str(a_client->name));
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int rc = 0;
	int argerr = 0;

	#ifdef HAVE_GETOPT_H
	int option_index = 0;
	/* INDENT-OFF */
	static struct option long_options[] = {
	{"per-action-cib", 0, 0, 'a'},
	{"stand-alone", 0, 0, 's'},
	{"disk-writes", 0, 0, 'w'},

	{"cib-root", 1, 0, 'r'},

	{"verbose", 0, 0, 'V'},
	{"help", 0, 0, '?'},
	{"metadata", 0, 0, 'm'},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */
	#endif

	struct passwd *pwentry = NULL;

	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	mainloop_add_signal(SIGTERM, cib_shutdown);
	mainloop_add_signal(SIGPIPE, cib_enable_writes);

	cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL);

	crm_peer_init();
	client_list = g_hash_table_new(crm_str_hash, g_str_equal);

	while (1) {
	#ifdef HAVE_GETOPT_H
	flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index);
	#else
	flag = getopt(argc, argv, OPTARGS);
	#endif
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level();
	break;
	case 's':
	stand_alone = TRUE;
	preserve_status = TRUE;
	cib_writes_enabled = FALSE;

	pwentry = getpwnam(CRM_DAEMON_USER);
	CRM_CHECK(pwentry != NULL,
	crm_perror(LOG_ERR, "Invalid uid (%s) specified", CRM_DAEMON_USER);
	return 100);

	rc = setgid(pwentry->pw_gid);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not set group to %d", pwentry->pw_gid);
	return 100;
	}

	rc = setuid(pwentry->pw_uid);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not set user to %d", pwentry->pw_uid);
	return 100;
	}
	break;
	case '?': /* Help message */
	usage(crm_system_name, EX_OK);
	break;
	case 'w':
	cib_writes_enabled = TRUE;
	break;
	case 'r':
	cib_root = optarg;
	break;
	case 'm':
	cib_metadata();
	return 0;
	default:
	++argerr;
	break;
	}
	}
	if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) {
	cib_metadata();
	return 0;
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	usage(crm_system_name, EX_USAGE);
	}

	if (crm_is_writable(cib_root, NULL, CRM_DAEMON_USER, CRM_DAEMON_GROUP, FALSE) == FALSE) {
	crm_err("Bad permissions on %s. Terminating", cib_root);
	fprintf(stderr, "ERROR: Bad permissions on %s. See logs for details\n", cib_root);
	fflush(stderr);
	return 100;
	}

	/* read local config file */
	rc = cib_init();

	CRM_CHECK(g_hash_table_size(client_list) == 0, crm_warn("Not all clients gone at exit"));
	g_hash_table_foreach(client_list, log_cib_client, NULL);
	cib_cleanup();

	#if SUPPORT_HEARTBEAT
	if (hb_conn) {
	hb_conn->llc_ops->delete(hb_conn);
	}
	#endif

	crm_info("Done");
	return rc;
	}

	void
	cib_cleanup(void)
	{
	crm_peer_destroy();
	if (local_notify_queue) {
	g_hash_table_destroy(local_notify_queue);
	}
	g_hash_table_destroy(config_hash);
	g_hash_table_destroy(client_list);
	free(cib_our_uname);
	#if HAVE_LIBXML2
	crm_xml_cleanup();
	#endif
	free(channel1);
	free(channel2);
	free(channel3);
	free(channel4);
	free(channel5);
	}

	unsigned long cib_num_ops = 0;
	const char *cib_stat_interval = "10min";
	unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0;
	unsigned long cib_bad_connects = 0, cib_num_timeouts = 0;

	#if SUPPORT_HEARTBEAT
	gboolean ccm_connect(void);

	static void
	ccm_connection_destroy(gpointer user_data)
	{
	crm_err("CCM connection failed... blocking while we reconnect");
	CRM_ASSERT(ccm_connect());
	return;
	}

	static void *ccm_library = NULL;

	gboolean
	ccm_connect(void)
	{
	gboolean did_fail = TRUE;
	int num_ccm_fails = 0;
	int max_ccm_fails = 30;
	int ret;
	int cib_ev_fd;

	int (ccm_api_register) (oc_ev_t * token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1);

	int (ccm_api_set_callback) (const oc_ev_t token,
	oc_ev_class_t class,
	oc_ev_callback_t * fn,
	oc_ev_callback_t ** prev_fn) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1);

	void (ccm_api_special) (const oc_ev_t , oc_ev_class_t, int) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1);
	int (ccm_api_activate) (const oc_ev_t token, int *fd) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1);
	int (ccm_api_unregister) (oc_ev_t token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister", 1);

	static struct mainloop_fd_callbacks ccm_fd_callbacks =
	{
	.dispatch = cib_ccm_dispatch,
	.destroy = ccm_connection_destroy,
	};

	while (did_fail) {
	did_fail = FALSE;
	crm_info("Registering with CCM...");
	ret = (*ccm_api_register) (&cib_ev_token);
	if (ret != 0) {
	did_fail = TRUE;
	}

	if (did_fail == FALSE) {
	crm_trace("Setting up CCM callbacks");
	ret = (*ccm_api_set_callback) (cib_ev_token, OC_EV_MEMB_CLASS,
	cib_ccm_msg_callback, NULL);
	if (ret != 0) {
	crm_warn("CCM callback not set");
	did_fail = TRUE;
	}
	}
	if (did_fail == FALSE) {
	(*ccm_api_special) (cib_ev_token, OC_EV_MEMB_CLASS, 0);

	crm_trace("Activating CCM token");
	ret = (*ccm_api_activate) (cib_ev_token, &cib_ev_fd);
	if (ret != 0) {
	crm_warn("CCM Activation failed");
	did_fail = TRUE;
	}
	}

	if (did_fail) {
	num_ccm_fails++;
	(*ccm_api_unregister) (cib_ev_token);

	if (num_ccm_fails < max_ccm_fails) {
	crm_warn("CCM Connection failed %d times (%d max)", num_ccm_fails, max_ccm_fails);
	sleep(3);

	} else {
	crm_err("CCM Activation failed %d (max) times", num_ccm_fails);
	return FALSE;
	}
	}
	}

	crm_debug("CCM Activation passed... all set to go!");
	- mainloop_add_fd("heartbeat-ccm", cib_ev_fd, cib_ev_token, &ccm_fd_callbacks);
	+ mainloop_add_fd("heartbeat-ccm", G_PRIORITY_MEDIUM, cib_ev_fd, cib_ev_token, &ccm_fd_callbacks);

	return TRUE;
	}
	#endif

	#if SUPPORT_COROSYNC
	static gboolean
	cib_ais_dispatch(AIS_Message * wrapper, char *data, int sender)
	{
	xmlNode *xml = NULL;

	if (wrapper->header.id == crm_class_cluster) {
	xml = string2xml(data);
	if (xml == NULL) {
	goto bail;
	}
	crm_xml_add(xml, F_ORIG, wrapper->sender.uname);
	crm_xml_add_int(xml, F_SEQ, wrapper->id);
	cib_peer_callback(xml, NULL);
	}

	free_xml(xml);
	return TRUE;

	bail:
	crm_err("Invalid XML: '%.120s'", data);
	return TRUE;

	}

	static void
	cib_ais_destroy(gpointer user_data)
	{
	if (cib_shutdown_flag) {
	crm_info("Corosync disconnection complete");
	} else {
	crm_err("Corosync connection lost! Exiting.");
	terminate_cib(__FUNCTION__, TRUE);
	}
	}
	#endif

	static void
	cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	#if 0
	/* crm_active_peers(crm_proc_cib) appears to give the wrong answer
	* sometimes, this might help figure out why
	*/
	if(type == crm_status_nstate) {
	crm_info("status: %s is now %s (was %s)", node->uname, node->state, (const char *)data);
	if (safe_str_eq(CRMD_JOINSTATE_MEMBER, node->state)) {
	return;
	}

	} else if(type == crm_status_processes) {
	uint32_t old = 0;
	if (data) {
	old = (const uint32_t )data;
	}

	if ((node->processes ^ old) & crm_proc_cib) {
	crm_info("status: cib process on %s is now %sactive",
	node->uname, is_set(node->processes, crm_proc_cib)?"":"in");
	} else {
	return;
	}

	} else {
	return;
	}
	#endif
	if(cib_shutdown_flag && crm_active_peers() < 2 && g_hash_table_size(client_list) == 0) {
	crm_info("No more peers");
	terminate_cib(__FUNCTION__, FALSE);
	}
	}

	#if SUPPORT_HEARTBEAT
	static void
	cib_ha_connection_destroy(gpointer user_data)
	{
	if (cib_shutdown_flag) {
	crm_info("Heartbeat disconnection complete... exiting");
	terminate_cib(__FUNCTION__, FALSE);
	} else {
	crm_err("Heartbeat connection lost! Exiting.");
	terminate_cib(__FUNCTION__, TRUE);
	}
	}
	#endif

	int
	cib_init(void)
	{
	config_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (startCib("cib.xml") == FALSE) {
	crm_crit("Cannot start CIB... terminating");
	exit(1);
	}

	if (stand_alone == FALSE) {
	void *dispatch = NULL;
	void *destroy = NULL;

	if (is_openais_cluster()) {
	#if SUPPORT_COROSYNC
	destroy = cib_ais_destroy;
	dispatch = cib_ais_dispatch;
	#endif
	} else if(is_heartbeat_cluster()) {
	#if SUPPORT_HEARTBEAT
	dispatch = cib_ha_peer_callback;
	destroy = cib_ha_connection_destroy;
	#endif
	}

	if (crm_cluster_connect(&cib_our_uname, NULL, dispatch, destroy,
	#if SUPPORT_HEARTBEAT
	&hb_conn
	#else
	NULL
	#endif
	) == FALSE) {
	crm_crit("Cannot sign in to the cluster... terminating");
	exit(100);
	}
	if (is_openais_cluster()) {
	crm_set_status_callback(&cib_peer_update_callback);
	}
	#if SUPPORT_HEARTBEAT
	if (is_heartbeat_cluster()) {

	gboolean was_error = FALSE;

	if (was_error == FALSE) {
	if (HA_OK !=
	hb_conn->llc_ops->set_cstatus_callback(hb_conn, cib_client_status_callback,
	hb_conn)) {

	crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn));
	was_error = TRUE;
	}
	}

	if (was_error == FALSE) {
	was_error = (ccm_connect() == FALSE);
	}

	if (was_error == FALSE) {
	/* Async get client status information in the cluster */
	crm_info("Requesting the list of configured nodes");
	hb_conn->llc_ops->client_status(hb_conn, NULL, CRM_SYSTEM_CIB, -1);
	}
	}
	#endif

	} else {
	cib_our_uname = strdup("localhost");
	}

	ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, &ipc_ro_callbacks);
	ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, &ipc_rw_callbacks);
	ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, &ipc_rw_callbacks);

	if (stand_alone) {
	cib_is_master = TRUE;
	}

	if (ipcs_ro != NULL && ipcs_rw != NULL && ipcs_shm != NULL) {
	/* Create the mainloop and run it... */
	mainloop = g_main_new(FALSE);
	crm_info("Starting %s mainloop", crm_system_name);

	g_main_run(mainloop);

	} else {
	crm_err("Couldnt start all IPC channels, exiting.");
	return -1;
	}

	qb_ipcs_destroy(ipcs_ro);
	qb_ipcs_destroy(ipcs_rw);
	qb_ipcs_destroy(ipcs_shm);

	return 0;
	}

	void
	usage(const char *cmd, int exit_status)
	{
	FILE *stream;

	stream = exit_status ? stderr : stdout;

	fprintf(stream, "usage: %s [-%s]\n", cmd, OPTARGS);
	fprintf(stream, "\t--%s (-%c)\t\tTurn on debug info."
	" Additional instances increase verbosity\n", "verbose", 'V');
	fprintf(stream, "\t--%s (-%c)\t\tThis help message\n", "help", '?');
	fprintf(stream, "\t--%s (-%c)\t\tShow configurable cib options\n", "metadata", 'm');
	fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "per-action-cib", 'a');
	fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "stand-alone", 's');
	fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "disk-writes", 'w');
	fprintf(stream, "\t--%s (-%c)\t\tAdvanced use only\n", "cib-root", 'r');
	fflush(stream);

	exit(exit_status);
	}

	gboolean
	startCib(const char *filename)
	{
	gboolean active = FALSE;
	xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status);

	CRM_ASSERT(cib != NULL);

	if (activateCibXml(cib, TRUE, "start") == 0) {
	int port = 0;
	const char *port_s = NULL;

	active = TRUE;

	cib_read_config(config_hash, cib);

	port_s = crm_element_value(cib, "remote-tls-port");
	if (port_s) {
	port = crm_parse_int(port_s, "0");
	remote_tls_fd = init_remote_listener(port, TRUE);
	}

	port_s = crm_element_value(cib, "remote-clear-port");
	if (port_s) {
	port = crm_parse_int(port_s, "0");
	remote_fd = init_remote_listener(port, FALSE);
	}

	crm_info("CIB Initialization completed successfully");
	}

	return active;
	}
	diff --git a/cib/remote.c b/cib/remote.c
	index c8a43a9b64..e8c2398c89 100644
	--- a/cib/remote.c
	+++ b/cib/remote.c
	@@ -1,608 +1,608 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>

	#include <netinet/ip.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>

	#include "callbacks.h"
	/* #undef HAVE_PAM_PAM_APPL_H */
	/* #undef HAVE_GNUTLS_GNUTLS_H */

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <pwd.h>
	#include <grp.h>
	#if HAVE_SECURITY_PAM_APPL_H
	# include <security/pam_appl.h>
	# define HAVE_PAM 1
	#else
	# if HAVE_PAM_PAM_APPL_H
	# include <pam/pam_appl.h>
	# define HAVE_PAM 1
	# endif
	#endif

	#ifdef HAVE_DECL_NANOSLEEP
	# include <time.h>
	#endif

	extern int remote_tls_fd;
	extern gboolean cib_shutdown_flag;

	int init_remote_listener(int port, gboolean encrypted);
	void cib_remote_connection_destroy(gpointer user_data);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define DH_BITS 1024
	gnutls_dh_params dh_params;
	extern gnutls_anon_server_credentials anon_cred_s;
	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}

	extern gnutls_session *create_tls_session(int csock, int type);

	#endif

	int num_clients;
	int authenticate_user(const char user, const char passwd);
	int cib_remote_listen(gpointer data);
	int cib_remote_msg(gpointer data);

	static void
	remote_connection_destroy(gpointer user_data)
	{
	return;
	}

	#define ERROR_SUFFIX " Shutting down remote listener"
	int
	init_remote_listener(int port, gboolean encrypted)
	{
	int ssock;
	struct sockaddr_in saddr;
	int optval;
	static struct mainloop_fd_callbacks remote_listen_fd_callbacks =
	{
	.dispatch = cib_remote_listen,
	.destroy = remote_connection_destroy,
	};

	if (port <= 0) {
	/* dont start it */
	return 0;
	}

	if (encrypted) {
	#ifndef HAVE_GNUTLS_GNUTLS_H
	crm_warn("TLS support is not available");
	return 0;
	#else
	crm_notice("Starting a tls listener on port %d.", port);
	gnutls_global_init();
	/* gnutls_global_set_log_level (10); */
	gnutls_global_set_log_function(debug_log);
	gnutls_dh_params_init(&dh_params);
	gnutls_dh_params_generate2(dh_params, DH_BITS);
	gnutls_anon_allocate_server_credentials(&anon_cred_s);
	gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
	#endif
	} else {
	crm_warn("Starting a plain_text listener on port %d.", port);
	}
	#ifndef HAVE_PAM
	crm_warn("PAM is _not_ enabled!");
	#endif

	/* create server socket */
	ssock = socket(AF_INET, SOCK_STREAM, 0);
	if (ssock == -1) {
	crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
	return -1;
	}

	/* reuse address */
	optval = 1;
	setsockopt(ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));

	/* bind server socket */
	memset(&saddr, '\0', sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_addr.s_addr = INADDR_ANY;
	saddr.sin_port = htons(port);
	if (bind(ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
	crm_perror(LOG_ERR, "Can not bind server socket." ERROR_SUFFIX);
	close(ssock);
	return -2;
	}
	if (listen(ssock, 10) == -1) {
	crm_perror(LOG_ERR, "Can not start listen." ERROR_SUFFIX);
	close(ssock);
	return -3;
	}

	- mainloop_add_fd("cib-remote", ssock, &ssock, &remote_listen_fd_callbacks);
	+ mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, ssock, &ssock, &remote_listen_fd_callbacks);

	return ssock;
	}

	static int
	check_group_membership(const char usr, const char grp)
	{
	int index = 0;
	struct passwd *pwd = NULL;
	struct group *group = NULL;

	CRM_CHECK(usr != NULL, return FALSE);
	CRM_CHECK(grp != NULL, return FALSE);

	pwd = getpwnam(usr);
	if (pwd == NULL) {
	crm_err("No user named '%s' exists!", usr);
	return FALSE;
	}

	group = getgrgid(pwd->pw_gid);
	if (group != NULL && crm_str_eq(grp, group->gr_name, TRUE)) {
	return TRUE;
	}

	group = getgrnam(grp);
	if (group == NULL) {
	crm_err("No group named '%s' exists!", grp);
	return FALSE;
	}

	while (TRUE) {
	char *member = group->gr_mem[index++];

	if (member == NULL) {
	break;

	} else if (crm_str_eq(usr, member, TRUE)) {
	return TRUE;
	}
	};

	return FALSE;
	}

	int
	cib_remote_listen(gpointer data)
	{
	int lpc = 0;
	int csock = 0;
	unsigned laddr;
	time_t now = 0;
	time_t start = time(NULL);
	struct sockaddr_in addr;
	int ssock = (int )data;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_session *session = NULL;
	#endif
	cib_client_t *new_client = NULL;

	xmlNode *login = NULL;
	const char *user = NULL;
	const char *pass = NULL;
	const char *tmp = NULL;

	#ifdef HAVE_DECL_NANOSLEEP
	const struct timespec sleepfast = { 0, 10000000 }; /* 10 millisec */
	#endif

	static struct mainloop_fd_callbacks remote_client_fd_callbacks =
	{
	.dispatch = cib_remote_msg,
	.destroy = cib_remote_connection_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_debug("New %s connection from %s",
	ssock == remote_tls_fd ? "secure" : "clear-text", inet_ntoa(addr.sin_addr));

	if (csock == -1) {
	crm_err("accept socket failed");
	return TRUE;
	}

	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	/* create gnutls session for the server socket */
	session = create_tls_session(csock, GNUTLS_SERVER);
	if (session == NULL) {
	crm_err("TLS session creation failed");
	close(csock);
	return TRUE;
	}
	#endif
	}

	do {
	crm_trace("Iter: %d", lpc++);
	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	login = crm_recv_remote_msg(session, TRUE);
	#endif
	} else {
	login = crm_recv_remote_msg(GINT_TO_POINTER(csock), FALSE);
	}
	if (login != NULL) {
	break;
	}
	#ifdef HAVE_DECL_NANOSLEEP
	nanosleep(&sleepfast, NULL);
	#else
	sleep(1);
	#endif
	now = time(NULL);

	/* Peers have 3s to connect */
	} while (login == NULL && (start - now) < 4);

	crm_log_xml_info(login, "Login: ");
	if (login == NULL) {
	goto bail;
	}

	tmp = crm_element_name(login);
	if (safe_str_neq(tmp, "cib_command")) {
	crm_err("Wrong tag: %s", tmp);
	goto bail;
	}

	tmp = crm_element_value(login, "op");
	if (safe_str_neq(tmp, "authenticate")) {
	crm_err("Wrong operation: %s", tmp);
	goto bail;
	}

	user = crm_element_value(login, "user");
	pass = crm_element_value(login, "password");

	/* Non-root daemons can only validate the password of the
	* user they're running as
	*/
	if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
	crm_err("User is not a member of the required group");
	goto bail;

	} else if (authenticate_user(user, pass) == FALSE) {
	crm_err("PAM auth failed");
	goto bail;
	}

	/* send ACK */
	num_clients++;
	new_client = calloc(1, sizeof(cib_client_t));
	new_client->name = crm_element_value_copy(login, "name");

	CRM_CHECK(new_client->id == NULL, free(new_client->id));
	new_client->id = crm_generate_uuid();

	#if ENABLE_ACL
	new_client->user = strdup(user);
	#endif

	new_client->callback_id = NULL;
	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	new_client->encrypted = TRUE;
	new_client->session = session;
	#endif
	} else {
	new_client->session = GINT_TO_POINTER(csock);
	}

	free_xml(login);
	login = create_xml_node(NULL, "cib_result");
	crm_xml_add(login, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(login, F_CIB_CLIENTID, new_client->id);
	crm_send_remote_msg(new_client->session, login, new_client->encrypted);
	free_xml(login);

	new_client->remote = mainloop_add_fd(
	- "cib-remote-client", csock, new_client, &remote_client_fd_callbacks);
	+ "cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &remote_client_fd_callbacks);

	g_hash_table_insert(client_list, new_client->id, new_client);

	return TRUE;

	bail:
	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_bye(*session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*session);
	gnutls_free(session);
	#endif
	}
	close(csock);
	free_xml(login);
	return TRUE;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	cib_client_t *client = user_data;

	if (client == NULL) {
	return;
	}

	crm_trace("Cleaning up after client disconnect: %s/%s",
	crm_str(client->name), client->id);

	if (client->id != NULL) {
	if (!g_hash_table_remove(client_list, client->id)) {
	crm_err("Client %s not found in the hashtable", client->name);
	}
	}

	crm_trace("Destroying %s (%p)", client->name, user_data);
	num_clients--;
	crm_trace("Num unfree'd clients: %d", num_clients);
	free(client->name);
	free(client->callback_id);
	free(client->id);
	free(client->user);
	free(client);
	crm_trace("Freed the cib client");

	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	return;
	}

	int
	cib_remote_msg(gpointer data)
	{
	const char *value = NULL;
	xmlNode *command = NULL;
	cib_client_t *client = data;

	crm_trace("%s callback", client->encrypted ? "secure" : "clear-text");

	command = crm_recv_remote_msg(client->session, client->encrypted);
	if (command == NULL) {
	return -1;
	}

	value = crm_element_name(command);
	if (safe_str_neq(value, "cib_command")) {
	crm_log_xml_trace(command, "Bad command: ");
	goto bail;
	}

	if (client->name == NULL) {
	value = crm_element_value(command, F_CLIENTNAME);
	if (value == NULL) {
	client->name = strdup(client->id);
	} else {
	client->name = strdup(value);
	}
	}

	if (client->callback_id == NULL) {
	value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
	if (value != NULL) {
	client->callback_id = strdup(value);
	crm_trace("Callback channel for %s is %s", client->id, client->callback_id);

	} else {
	client->callback_id = strdup(client->id);
	}
	}

	/* unset dangerous options */
	xml_remove_prop(command, F_ORIG);
	xml_remove_prop(command, F_CIB_HOST);
	xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);

	crm_xml_add(command, F_TYPE, T_CIB);
	crm_xml_add(command, F_CIB_CLIENTID, client->id);
	crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
	#if ENABLE_ACL
	crm_xml_add(command, F_CIB_USER, client->user);
	#endif

	if (crm_element_value(command, F_CIB_CALLID) == NULL) {
	char *call_uuid = crm_generate_uuid();

	/* fix the command */
	crm_xml_add(command, F_CIB_CALLID, call_uuid);
	free(call_uuid);
	}

	if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
	crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
	}

	crm_log_xml_trace(command, "Remote command: ");
	cib_common_callback_worker(command, client, TRUE);
	bail:
	free_xml(command);
	command = NULL;
	return 0;
	}

	#ifdef HAVE_PAM
	/*
	* Useful Examples:
	* http://www.kernel.org/pub/linux/libs/pam/Linux-PAM-html
	* http://developer.apple.com/samplecode/CryptNoMore/index.html
	*/
	static int
	construct_pam_passwd(int num_msg, const struct pam_message **msg,
	struct pam_response *response, void data)
	{
	int count = 0;
	struct pam_response *reply;
	char string = (char )data;

	CRM_CHECK(data, return PAM_CONV_ERR);
	CRM_CHECK(num_msg == 1, return PAM_CONV_ERR); /* We only want to handle one message */

	reply = calloc(1, sizeof(struct pam_response));
	CRM_ASSERT(reply != NULL);

	for (count = 0; count < num_msg; ++count) {
	switch (msg[count]->msg_style) {
	case PAM_TEXT_INFO:
	crm_info("PAM: %s\n", msg[count]->msg);
	break;
	case PAM_PROMPT_ECHO_OFF:
	case PAM_PROMPT_ECHO_ON:
	reply[count].resp_retcode = 0;
	reply[count].resp = string; /* We already made a copy */
	case PAM_ERROR_MSG:
	/* In theory we'd want to print this, but then
	* we see the password prompt in the logs
	*/
	/* crm_err("PAM error: %s\n", msg[count]->msg); */
	break;
	default:
	crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
	goto bail;
	}
	}

	*response = reply;
	reply = NULL;

	return PAM_SUCCESS;

	bail:
	for (count = 0; count < num_msg; ++count) {
	if (reply[count].resp != NULL) {
	switch (msg[count]->msg_style) {
	case PAM_PROMPT_ECHO_ON:
	case PAM_PROMPT_ECHO_OFF:
	/* Erase the data - it contained a password */
	while (*(reply[count].resp)) {
	*(reply[count].resp)++ = '\0';
	}
	free(reply[count].resp);
	break;
	}
	reply[count].resp = NULL;
	}
	}
	free(reply);
	reply = NULL;

	return PAM_CONV_ERR;
	}
	#endif

	int
	authenticate_user(const char user, const char passwd)
	{
	#ifndef HAVE_PAM
	gboolean pass = TRUE;
	#else
	int rc = 0;
	gboolean pass = FALSE;
	const void *p_user = NULL;

	struct pam_conv p_conv;
	struct pam_handle *pam_h = NULL;
	static const char *pam_name = NULL;

	if (pam_name == NULL) {
	pam_name = getenv("CIB_pam_service");
	}
	if (pam_name == NULL) {
	pam_name = "login";
	}

	p_conv.conv = construct_pam_passwd;
	p_conv.appdata_ptr = strdup(passwd);

	rc = pam_start(pam_name, user, &p_conv, &pam_h);
	if (rc != PAM_SUCCESS) {
	crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	rc = pam_authenticate(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	/* Make sure we authenticated the user we wanted to authenticate.
	* Since we also run as non-root, it might be worth pre-checking
	* the user has the same EID as us, since that the only user we
	* can authenticate.
	*/
	rc = pam_get_item(pam_h, PAM_USER, &p_user);
	if (rc != PAM_SUCCESS) {
	crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;

	} else if (p_user == NULL) {
	crm_err("Unknown user authenticated.");
	goto bail;

	} else if (safe_str_neq(p_user, user)) {
	crm_err("User mismatch: %s vs. %s.", (const char )p_user, (const char )user);
	goto bail;
	}

	rc = pam_acct_mgmt(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}
	pass = TRUE;

	bail:
	rc = pam_end(pam_h, rc);
	#endif
	return pass;
	}
	diff --git a/crmd/heartbeat.c b/crmd/heartbeat.c
	index 3a13e9948f..cae143b009 100644
	--- a/crmd/heartbeat.c
	+++ b/crmd/heartbeat.c
	@@ -1,533 +1,533 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	/* put these first so that uuid_t is defined without conflicts */
	#include <crm_internal.h>

	#include <string.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crmd_messages.h>
	#include <crmd_fsa.h>
	#include <fsa_proto.h>
	#include <crmd_callbacks.h>
	#include <tengine.h>
	#include <membership.h>


	#include <ocf/oc_event.h>
	#include <ocf/oc_membership.h>

	void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int);
	void ccm_event_detail(const oc_ev_membership_t * oc, oc_ed_t event);
	gboolean crmd_ha_msg_dispatch(ll_cluster_t * cluster_conn, gpointer user_data);
	void crmd_ccm_msg_callback(oc_ed_t event, void cookie, size_t size, const void data);
	int ccm_dispatch(gpointer user_data);

	#define CCM_EVENT_DETAIL 0
	#define CCM_EVENT_DETAIL_PARTIAL 0

	int (ccm_api_callback_done) (void cookie) = NULL;
	int (ccm_api_handle_event) (const oc_ev_t token) = NULL;
	static gboolean fsa_have_quorum = FALSE;

	static oc_ev_t *fsa_ev_token;
	static void *ccm_library = NULL;
	static int num_ccm_register_fails = 0;
	static int max_ccm_register_fails = 30;

	static void
	ccm_connection_destroy(void *userdata)
	{
	}

	/* A_CCM_CONNECT */
	void
	do_ccm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static struct mainloop_fd_callbacks ccm_fd_callbacks =
	{
	.dispatch = ccm_dispatch,
	.destroy = ccm_connection_destroy,
	};

	if (is_heartbeat_cluster()) {
	int (ccm_api_register) (oc_ev_t * token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1);

	int (ccm_api_set_callback) (const oc_ev_t token,
	oc_ev_class_t class,
	oc_ev_callback_t * fn,
	oc_ev_callback_t ** prev_fn) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1);

	void (ccm_api_special) (const oc_ev_t , oc_ev_class_t, int) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1);
	int (ccm_api_activate) (const oc_ev_t token, int *fd) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1);
	int (ccm_api_unregister) (oc_ev_t token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister", 1);

	if (action & A_CCM_DISCONNECT) {
	set_bit(fsa_input_register, R_CCM_DISCONNECTED);
	(*ccm_api_unregister) (fsa_ev_token);
	}

	if (action & A_CCM_CONNECT) {
	int ret;
	int fsa_ev_fd;
	gboolean did_fail = FALSE;

	crm_trace("Registering with CCM");
	clear_bit(fsa_input_register, R_CCM_DISCONNECTED);
	ret = (*ccm_api_register) (&fsa_ev_token);
	if (ret != 0) {
	crm_warn("CCM registration failed");
	did_fail = TRUE;
	}

	if (did_fail == FALSE) {
	crm_trace("Setting up CCM callbacks");
	ret = (*ccm_api_set_callback) (fsa_ev_token, OC_EV_MEMB_CLASS,
	crmd_ccm_msg_callback, NULL);
	if (ret != 0) {
	crm_warn("CCM callback not set");
	did_fail = TRUE;
	}
	}
	if (did_fail == FALSE) {
	(ccm_api_special) (fsa_ev_token, OC_EV_MEMB_CLASS, 0 /don't care */ );

	crm_trace("Activating CCM token");
	ret = (*ccm_api_activate) (fsa_ev_token, &fsa_ev_fd);
	if (ret != 0) {
	crm_warn("CCM Activation failed");
	did_fail = TRUE;
	}
	}

	if (did_fail) {
	num_ccm_register_fails++;
	(*ccm_api_unregister) (fsa_ev_token);

	if (num_ccm_register_fails < max_ccm_register_fails) {
	crm_warn("CCM Connection failed"
	" %d times (%d max)", num_ccm_register_fails, max_ccm_register_fails);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(NULL);
	return;

	} else {
	crm_err("CCM Activation failed %d (max) times", num_ccm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	return;
	}
	}

	crm_info("CCM connection established... waiting for first callback");
	- mainloop_add_fd("heartbeat-ccm", fsa_ev_fd, fsa_ev_token, &ccm_fd_callbacks);
	+ mainloop_add_fd("heartbeat-ccm", G_PRIORITY_HIGH, fsa_ev_fd, fsa_ev_token, &ccm_fd_callbacks);

	}
	}

	if (action & ~(A_CCM_CONNECT \| A_CCM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	void
	ccm_event_detail(const oc_ev_membership_t * oc, oc_ed_t event)
	{
	int lpc;
	gboolean member = FALSE;

	member = FALSE;

	crm_trace("-----------------------");
	crm_info("%s: trans=%d, nodes=%d, new=%d, lost=%d n_idx=%d, "
	"new_idx=%d, old_idx=%d",
	ccm_event_name(event),
	oc->m_instance,
	oc->m_n_member, oc->m_n_in, oc->m_n_out, oc->m_memb_idx, oc->m_in_idx, oc->m_out_idx);

	# if !CCM_EVENT_DETAIL_PARTIAL
	for (lpc = 0; lpc < oc->m_n_member; lpc++) {
	crm_info("\tCURRENT: %s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_memb_idx + lpc].node_uname,
	oc->m_array[oc->m_memb_idx + lpc].node_id,
	oc->m_array[oc->m_memb_idx + lpc].node_born_on);

	if (safe_str_eq(fsa_our_uname, oc->m_array[oc->m_memb_idx + lpc].node_uname)) {
	member = TRUE;
	}
	}
	if (member == FALSE) {
	crm_warn("MY NODE IS NOT IN CCM THE MEMBERSHIP LIST");
	}
	# endif
	for (lpc = 0; lpc < (int)oc->m_n_in; lpc++) {
	crm_info("\tNEW: %s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_in_idx + lpc].node_uname,
	oc->m_array[oc->m_in_idx + lpc].node_id,
	oc->m_array[oc->m_in_idx + lpc].node_born_on);
	}

	for (lpc = 0; lpc < (int)oc->m_n_out; lpc++) {
	crm_info("\tLOST: %s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_out_idx + lpc].node_uname,
	oc->m_array[oc->m_out_idx + lpc].node_id,
	oc->m_array[oc->m_out_idx + lpc].node_born_on);
	}

	crm_trace("-----------------------");

	}


	/* A_CCM_UPDATE_CACHE */
	/*
	* Take the opportunity to update the node status in the CIB as well
	*/
	void
	do_ccm_update_cache(enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state,
	oc_ed_t event, const oc_ev_membership_t * oc, xmlNode * xml)
	{
	unsigned long long instance = 0;
	unsigned int lpc = 0;

	if (is_heartbeat_cluster()) {
	CRM_ASSERT(oc != NULL);
	instance = oc->m_instance;
	}

	CRM_ASSERT(crm_peer_seq <= instance);

	switch (cur_state) {
	case S_STOPPING:
	case S_TERMINATE:
	case S_HALT:
	crm_debug("Ignoring %s CCM event %llu, we're in state %s",
	ccm_event_name(event), instance, fsa_state2string(cur_state));
	return;
	case S_ELECTION:
	register_fsa_action(A_ELECTION_CHECK);
	break;
	default:
	break;
	}

	if (is_heartbeat_cluster()) {
	ccm_event_detail(oc, event);

	/---- Recently Dead Member Nodes ----/
	for (lpc = 0; lpc < oc->m_n_out; lpc++) {
	crm_update_ccm_node(oc, lpc + oc->m_out_idx, CRM_NODE_LOST, instance);
	}

	/---- All Member Nodes ----/
	for (lpc = 0; lpc < oc->m_n_member; lpc++) {
	crm_update_ccm_node(oc, lpc + oc->m_memb_idx, CRM_NODE_ACTIVE, instance);
	}
	}

	if (event == OC_EV_MS_EVICTED) {
	crm_node_t *peer = crm_get_peer(0, fsa_our_uname);
	crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_EVICTED, 0);

	/* todo: drop back to S_PENDING instead */
	/* get out... NOW!
	*
	* go via the error recovery process so that HA will
	* restart us if required
	*/
	register_fsa_error_adv(cause, I_ERROR, NULL, NULL, __FUNCTION__);
	}

	post_cache_update(instance);
	return;
	}

	int
	ccm_dispatch(gpointer user_data)
	{
	int rc = 0;
	oc_ev_t ccm_token = (oc_ev_t ) user_data;
	gboolean was_error = FALSE;

	crm_trace("Invoked");
	if (ccm_api_handle_event == NULL) {
	ccm_api_handle_event =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_handle_event", 1);
	}
	rc = (*ccm_api_handle_event) (ccm_token);

	if (rc != 0) {
	if (is_set(fsa_input_register, R_CCM_DISCONNECTED) == FALSE) {
	/* we signed out, so this is expected */
	register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL);
	crm_err("CCM connection appears to have failed: rc=%d.", rc);
	}
	was_error = TRUE;
	}

	trigger_fsa(fsa_source);
	if(was_error) {
	return -1;
	}

	return 0;
	}

	void
	crmd_ccm_msg_callback(oc_ed_t event, void cookie, size_t size, const void data)
	{
	gboolean update_cache = FALSE;
	const oc_ev_membership_t *membership = data;

	gboolean update_quorum = FALSE;

	crm_trace("Invoked");
	CRM_ASSERT(data != NULL);

	crm_info("Quorum %s after event=%s (id=%d)",
	ccm_have_quorum(event) ? "(re)attained" : "lost",
	ccm_event_name(event), membership->m_instance);

	if (crm_peer_seq > membership->m_instance) {
	crm_err("Membership instance ID went backwards! %llu->%d",
	crm_peer_seq, membership->m_instance);
	CRM_ASSERT(crm_peer_seq <= membership->m_instance);
	return;
	}

	/*
	* OC_EV_MS_NEW_MEMBERSHIP: membership with quorum
	* OC_EV_MS_MS_INVALID: membership without quorum
	* OC_EV_MS_NOT_PRIMARY: previous membership no longer valid
	* OC_EV_MS_PRIMARY_RESTORED: previous membership restored
	* OC_EV_MS_EVICTED: the client is evicted from ccm.
	*/

	switch (event) {
	case OC_EV_MS_NEW_MEMBERSHIP:
	case OC_EV_MS_INVALID:
	update_cache = TRUE;
	update_quorum = TRUE;
	break;
	case OC_EV_MS_NOT_PRIMARY:
	break;
	case OC_EV_MS_PRIMARY_RESTORED:
	update_cache = TRUE;
	crm_peer_seq = membership->m_instance;
	break;
	case OC_EV_MS_EVICTED:
	update_quorum = TRUE;
	register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL);
	crm_err("Shutting down after CCM event: %s", ccm_event_name(event));
	break;
	default:
	crm_err("Unknown CCM event: %d", event);
	}

	if (update_quorum) {
	crm_have_quorum = ccm_have_quorum(event);
	crm_update_quorum(crm_have_quorum, FALSE);

	if (crm_have_quorum == FALSE) {
	/* did we just loose quorum? */
	if (fsa_have_quorum) {
	crm_info("Quorum lost: %s", ccm_event_name(event));
	}
	}
	}

	if (update_cache) {
	crm_trace("Updating cache after event %s", ccm_event_name(event));
	do_ccm_update_cache(C_CCM_CALLBACK, fsa_state, event, data, NULL);

	} else if (event != OC_EV_MS_NOT_PRIMARY) {
	crm_peer_seq = membership->m_instance;
	register_fsa_action(A_TE_CANCEL);
	}

	if (ccm_api_callback_done == NULL) {
	ccm_api_callback_done =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_callback_done", 1);
	}
	(*ccm_api_callback_done) (cookie);
	return;
	}


	void
	crmd_ha_status_callback(const char node, const char status, void *private)
	{
	xmlNode *update = NULL;
	crm_node_t *peer = NULL;

	crm_notice("Status update: Node %s now has status [%s]", node, status);

	peer = crm_get_peer(0, node);
	if (safe_str_eq(status, PINGSTATUS)) {
	return;
	}

	if (safe_str_eq(status, DEADSTATUS)) {
	/* this node is toast */
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_heartbeat, OFFLINESTATUS);

	} else {
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_heartbeat, ONLINESTATUS);
	}

	trigger_fsa(fsa_source);

	if(AM_I_DC) {
	update = do_update_node_cib(peer, node_update_cluster, NULL, __FUNCTION__);
	fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
	cib_scope_local \| cib_quorum_override \| cib_can_create);
	free_xml(update);
	}
	}

	void
	crmd_client_status_callback(const char node, const char client, const char status, void private)
	{
	crm_node_t *peer = NULL;

	crm_trace("Invoked");
	if (safe_str_neq(client, CRM_SYSTEM_CRMD)) {
	return;
	}

	set_bit(fsa_input_register, R_PEER_DATA);

	crm_notice("Status update: Client %s/%s now has status [%s] (DC=%s)",
	node, client, status, AM_I_DC ? "true" : "false");

	if (safe_str_eq(status, ONLINESTATUS)) {
	/* remove the cached value in case it changed */
	crm_trace("Uncaching UUID for %s", node);
	unget_uuid(node);
	}

	peer = crm_get_peer(0, node);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_crmd, status);

	if (AM_I_DC) {
	xmlNode *update = NULL;

	crm_trace("Got client status callback");
	update = do_update_node_cib(peer, node_update_peer, NULL, __FUNCTION__);
	fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
	cib_scope_local \| cib_quorum_override \| cib_can_create);
	free_xml(update);
	}
	}

	void
	crmd_ha_msg_callback(HA_Message * hamsg, void *private_data)
	{
	int level = LOG_DEBUG;
	crm_node_t *from_node = NULL;

	xmlNode *msg = convert_ha_message(NULL, hamsg, __FUNCTION__);
	const char *from = crm_element_value(msg, F_ORIG);
	const char *op = crm_element_value(msg, F_CRM_TASK);
	const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);

	CRM_CHECK(from != NULL, crm_log_xml_err(msg, "anon"); goto bail);

	crm_trace("HA[inbound]: %s from %s", op, from);

	if (crm_peer_cache == NULL \|\| crm_active_peers() == 0) {
	crm_debug("Ignoring HA messages until we are"
	" connected to the CCM (%s op from %s)", op, from);
	crm_log_xml_trace(msg, "HA[inbound]: Ignore (No CCM)");
	goto bail;
	}

	from_node = crm_get_peer(0, from);
	if (crm_is_peer_active(from_node) == FALSE) {
	if (safe_str_eq(op, CRM_OP_VOTE)) {
	level = LOG_WARNING;

	} else if (AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) {
	level = LOG_WARNING;

	} else if (safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
	level = LOG_WARNING;
	}
	do_crm_log(level,
	"Ignoring HA message (op=%s) from %s: not in our"
	" membership list (size=%d)", op, from, crm_active_peers());

	crm_log_xml_trace(msg, "HA[inbound]: CCM Discard");

	} else {
	crmd_ha_msg_filter(msg);
	}

	bail:
	free_xml(msg);
	return;
	}

	gboolean
	crmd_ha_msg_dispatch(ll_cluster_t * cluster_conn, gpointer user_data)
	{
	IPC_Channel *channel = NULL;
	gboolean stay_connected = TRUE;

	crm_trace("Invoked");

	if (cluster_conn != NULL) {
	channel = cluster_conn->llc_ops->ipcchan(cluster_conn);
	}

	CRM_CHECK(cluster_conn != NULL,;);
	CRM_CHECK(channel != NULL,;);

	if (channel != NULL && IPC_ISRCONN(channel)) {
	if (cluster_conn->llc_ops->msgready(cluster_conn) == 0) {
	crm_trace("no message ready yet");
	}
	/* invoke the callbacks but dont block */
	cluster_conn->llc_ops->rcvmsg(cluster_conn, 0);
	}

	if (channel == NULL \|\| channel->ch_status != IPC_CONNECT) {
	if (is_set(fsa_input_register, R_HA_DISCONNECTED) == FALSE) {
	crm_crit("Lost connection to heartbeat service.");
	} else {
	crm_info("Lost connection to heartbeat service.");
	}
	trigger_fsa(fsa_source);
	stay_connected = FALSE;
	}

	return stay_connected;
	}
	diff --git a/crmd/pengine.c b/crmd/pengine.c
	index b866898c14..e98f41096e 100644
	--- a/crmd/pengine.c
	+++ b/crmd/pengine.c
	@@ -1,291 +1,291 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <unistd.h> /* for access */

	#include <sys/types.h> /* for calls to open */
	#include <sys/stat.h> /* for calls to open */
	#include <fcntl.h> /* for calls to open */
	#include <pwd.h> /* for getpwuid */
	#include <grp.h> /* for initgroups */

	#include <sys/time.h> /* for getrlimit */
	#include <sys/resource.h> /* for getrlimit */

	#include <errno.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>

	#include <crm/cib.h>
	#include <crmd.h>

	struct crm_subsystem_s *pe_subsystem = NULL;
	void do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);

	static void
	save_cib_contents(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	char *id = user_data;

	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
	CRM_CHECK(id != NULL, return);

	if (rc == pcmk_ok) {
	int len = 15;
	char *filename = NULL;

	len += strlen(id);
	len += strlen(PE_STATE_DIR);

	filename = calloc(1, len);
	CRM_CHECK(filename != NULL, return);

	sprintf(filename, PE_STATE_DIR "/pe-core-%s.bz2", id);
	if (write_xml_file(output, filename, TRUE) < 0) {
	crm_err("Could not save CIB contents after PE crash to %s", filename);
	} else {
	crm_notice("Saved CIB contents after PE crash to %s", filename);
	}

	free(filename);
	}

	free(id);
	}

	static void
	pe_ipc_destroy(gpointer user_data)
	{
	clear_bit(fsa_input_register, pe_subsystem->flag_connected);
	if (is_set(fsa_input_register, pe_subsystem->flag_required)) {
	int rc = pcmk_ok;
	char *uuid_str = crm_generate_uuid();

	crm_crit("Connection to the Policy Engine failed (pid=%d, uuid=%s)",
	pe_subsystem->pid, uuid_str);

	/*
	*The PE died...
	*
	* Save the current CIB so that we have a chance of
	* figuring out what killed it.
	*
	* Delay raising the I_ERROR until the query below completes or
	* 5s is up, whichever comes first.
	*
	*/
	rc = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);
	fsa_cib_conn->cmds->register_callback(fsa_cib_conn, rc, 5, FALSE, uuid_str,
	"save_cib_contents", save_cib_contents);

	} else {
	crm_info("Connection to the Policy Engine released");
	}

	pe_subsystem->pid = -1;
	pe_subsystem->source = NULL;
	pe_subsystem->client = NULL;

	mainloop_set_trigger(fsa_source);
	return;
	}

	static int
	pe_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	if (msg) {
	route_message(C_IPC_MESSAGE, msg);
	}

	free_xml(msg);
	return 0;
	}

	/* A_PE_START, A_PE_STOP, A_TE_RESTART */
	void
	do_pe_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	struct crm_subsystem_s *this_subsys = pe_subsystem;

	long long stop_actions = A_PE_STOP;
	long long start_actions = A_PE_START;

	static struct ipc_client_callbacks pe_callbacks =
	{
	.dispatch = pe_ipc_dispatch,
	.destroy = pe_ipc_destroy
	};

	if (action & stop_actions) {
	clear_bit(fsa_input_register, pe_subsystem->flag_required);

	mainloop_del_ipc_client(pe_subsystem->source);
	pe_subsystem->source = NULL;

	clear_bit(fsa_input_register, pe_subsystem->flag_connected);
	}

	if ((action & start_actions) && (is_set(fsa_input_register, R_PE_CONNECTED) == FALSE)) {
	if (cur_state != S_STOPPING) {
	set_bit(fsa_input_register, pe_subsystem->flag_required);

	- pe_subsystem->source = mainloop_add_ipc_client(CRM_SYSTEM_PENGINE, 510241024/* 5Mb */, NULL, &pe_callbacks);
	+ pe_subsystem->source = mainloop_add_ipc_client(CRM_SYSTEM_PENGINE, G_PRIORITY_DEFAULT, 510241024/* 5Mb */, NULL, &pe_callbacks);

	if (pe_subsystem->source == NULL) {
	crm_warn("Setup of client connection failed, not adding channel to mainloop");
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	return;
	}

	/* if (is_openais_cluster()) { */
	/* pe_subsystem->pid = pe_subsystem->ipc->farside_pid; */
	/* } */

	set_bit(fsa_input_register, pe_subsystem->flag_connected);

	} else {
	crm_info("Ignoring request to start %s while shutting down", this_subsys->name);
	}
	}
	}

	int fsa_pe_query = 0;
	char *fsa_pe_ref = NULL;

	/* A_PE_INVOKE */
	void
	do_pe_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (AM_I_DC == FALSE) {
	crm_err("Not DC: No need to invoke the PE (anymore): %s", fsa_action2string(action));
	return;
	}

	if (is_set(fsa_input_register, R_PE_CONNECTED) == FALSE) {
	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_err("Cannot shut down gracefully without the PE");
	register_fsa_input_before(C_FSA_INTERNAL, I_TERMINATE, NULL);

	} else {
	crm_info("Waiting for the PE to connect");
	crmd_fsa_stall(NULL);
	register_fsa_action(A_PE_START);
	}
	return;
	}

	if (is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
	crm_err("Attempted to invoke the PE without a consistent" " copy of the CIB!");

	/* start the join from scratch */
	register_fsa_input_before(C_FSA_INTERNAL, I_ELECTION, NULL);
	return;
	}

	fsa_pe_query = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);

	crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query, fsa_state2string(fsa_state));

	/* Make sure any queued calculations are discarded */
	free(fsa_pe_ref);
	fsa_pe_ref = NULL;

	fsa_cib_conn->cmds->register_callback(fsa_cib_conn, fsa_pe_query, 60, FALSE, NULL,
	"do_pe_invoke_callback", do_pe_invoke_callback);
	}

	void
	do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	int sent;
	xmlNode *cmd = NULL;

	if (rc != pcmk_ok) {
	crm_err("Cant retrieve the CIB: %s (call %d)", pcmk_strerror(rc), call_id);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
	return;

	} else if (call_id != fsa_pe_query) {
	crm_trace("Skipping superceeded CIB query: %d (current=%d)", call_id, fsa_pe_query);
	return;

	} else if (AM_I_DC == FALSE \|\| is_set(fsa_input_register, R_PE_CONNECTED) == FALSE) {
	crm_debug("No need to invoke the PE anymore");
	return;

	} else if (fsa_state != S_POLICY_ENGINE) {
	crm_debug("Discarding PE request in state: %s", fsa_state2string(fsa_state));
	return;

	} else if (last_peer_update != 0) {
	crm_debug("Re-asking for the CIB: peer update %d still pending", last_peer_update);

	sleep(1);
	register_fsa_action(A_PE_INVOKE);
	return;

	} else if (fsa_state != S_POLICY_ENGINE) {
	crm_err("Invoking PE in state: %s", fsa_state2string(fsa_state));
	return;
	}

	CRM_LOG_ASSERT(output != NULL);

	crm_xml_add(output, XML_ATTR_DC_UUID, fsa_our_uuid);
	crm_xml_add_int(output, XML_ATTR_HAVE_QUORUM, fsa_has_quorum);

	if (ever_had_quorum && crm_have_quorum == FALSE) {
	crm_xml_add_int(output, XML_ATTR_QUORUM_PANIC, 1);
	}

	cmd = create_request(CRM_OP_PECALC, output, NULL, CRM_SYSTEM_PENGINE, CRM_SYSTEM_DC, NULL);

	free(fsa_pe_ref);
	fsa_pe_ref = crm_element_value_copy(cmd, XML_ATTR_REFERENCE);

	sent = crm_ipc_send(mainloop_get_ipc_client(pe_subsystem->source), cmd, NULL, 0);
	if (sent <= 0) {
	crm_err("Could not contact the pengine: %d", sent);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
	}

	crm_debug("Invoking the PE: query=%d, ref=%s, seq=%llu, quorate=%d",
	fsa_pe_query, fsa_pe_ref, crm_peer_seq, fsa_has_quorum);
	free_xml(cmd);
	}
	diff --git a/include/crm/common/mainloop.h b/include/crm/common/mainloop.h
	index 0a3dbe5e7e..5024b3ee70 100644
	--- a/include/crm/common/mainloop.h
	+++ b/include/crm/common/mainloop.h
	@@ -1,98 +1,100 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef CRM_COMMON_MAINLOOP__H
	# define CRM_COMMON_MAINLOOP__H

	# include <glib.h>

	typedef struct trigger_s crm_trigger_t;

	crm_trigger_t mainloop_add_trigger(int priority, int(dispatch) (gpointer user_data),
	gpointer userdata);

	void mainloop_set_trigger(crm_trigger_t * source);

	void mainloop_trigger_complete(crm_trigger_t *trig);

	gboolean mainloop_destroy_trigger(crm_trigger_t * source);

	gboolean crm_signal(int sig, void (*dispatch) (int sig));

	gboolean mainloop_add_signal(int sig, void (*dispatch) (int sig));

	gboolean mainloop_destroy_signal(int sig);

	#include <crm/common/ipc.h>

	struct ipc_client_callbacks
	{
	int (dispatch)(const char buffer, ssize_t length, gpointer userdata);
	void (*destroy) (gpointer);
	};

	qb_ipcs_service_t *mainloop_add_ipc_server(
	const char name, enum qb_ipc_type type, struct qb_ipcs_service_handlers callbacks);

	void mainloop_del_ipc_server(qb_ipcs_service_t *server);

	typedef struct mainloop_io_s mainloop_io_t;

	mainloop_io_t *mainloop_add_ipc_client(
	- const char name, size_t max_size, void userdata, struct ipc_client_callbacks *callbacks);
	+ const char name, int priority, size_t max_size, void userdata, struct ipc_client_callbacks *callbacks);

	void mainloop_del_ipc_client(mainloop_io_t *client);

	crm_ipc_t mainloop_get_ipc_client(mainloop_io_t client);


	struct mainloop_fd_callbacks
	{
	int (*dispatch)(gpointer userdata);
	void (*destroy)(gpointer userdata);
	};

	mainloop_io_t *mainloop_add_fd(
	- const char name, int fd, void userdata, struct mainloop_fd_callbacks *callbacks);
	+ const char name, int priority, int fd, void userdata, struct mainloop_fd_callbacks *callbacks);

	void mainloop_del_fd(mainloop_io_t *client);

	typedef struct mainloop_child_s mainloop_child_t;
	/*
	* Create a new tracked process
	* To track a process group, use -pid
	*/
	void
	mainloop_add_child(pid_t pid,
	int timeout,
	const char *desc,
	void *userdata,
	void (callback)(mainloop_child_t p,
	int status,
	int signo,
	int exitcode));

	void *
	mainloop_get_child_userdata(mainloop_child_t *child);
	int
	mainloop_get_child_timeout(mainloop_child_t *child);
	pid_t
	mainloop_get_child_pid(mainloop_child_t *child);
	void
	mainloop_clear_child_userdata(mainloop_child_t *child);

	+#define G_PRIORITY_MEDIUM (G_PRIORITY_HIGH/2)
	+
	#endif
	diff --git a/lib/cib/cib_native.c b/lib/cib/cib_native.c
	index a66aa696bf..f281f0fa04 100644
	--- a/lib/cib/cib_native.c
	+++ b/lib/cib/cib_native.c
	@@ -1,500 +1,500 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>

	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>

	typedef struct cib_native_opaque_s {
	char *token;
	crm_ipc_t *ipc;
	void (*dnotify_fn) (gpointer user_data);
	mainloop_io_t *source;

	} cib_native_opaque_t;

	int cib_native_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options);

	int cib_native_perform_op_delegate(cib_t * cib, const char op, const char host,
	const char section, xmlNode data, xmlNode ** output_data,
	int call_options, const char *user_name);

	int cib_native_free(cib_t * cib);
	int cib_native_signoff(cib_t * cib);
	int cib_native_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_native_signon_raw(cib_t * cib, const char name, enum cib_conn_type type, int event_fd);

	bool cib_native_dispatch(cib_t * cib);

	int cib_native_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data));

	cib_t *
	cib_native_new(void)
	{
	cib_native_opaque_t *native = NULL;
	cib_t *cib = cib_new_variant();

	native = calloc(1, sizeof(cib_native_opaque_t));

	cib->variant = cib_native;
	cib->variant_opaque = native;

	native->ipc = NULL;
	native->source = NULL;
	native->dnotify_fn = NULL;

	/* assign variant specific ops */
	cib->delegate_fn = cib_native_perform_op_delegate;
	cib->cmds->signon = cib_native_signon;
	cib->cmds->signon_raw = cib_native_signon_raw;
	cib->cmds->signoff = cib_native_signoff;
	cib->cmds->free = cib_native_free;

	cib->cmds->register_notification = cib_native_register_notification;
	cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify;

	return cib;
	}

	int
	cib_native_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	return cib_native_signon_raw(cib, name, type, NULL);
	}

	static int
	cib_native_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	xmlNode *msg = NULL;

	cib_t * cib = userdata;
	cib_native_opaque_t *native;

	crm_trace("dispatching %p", userdata);

	if (cib == NULL) {
	crm_err("No CIB!");
	return 0;
	}

	native = cib->variant_opaque;
	msg = string2xml(buffer);

	if (msg == NULL) {
	crm_warn("Received a NULL msg from CIB service.");
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(msg, F_TYPE);
	crm_trace("Activating %s callbacks...", type);
	crm_log_xml_trace(msg, "cib-reply");

	if (safe_str_eq(type, T_CIB)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (safe_str_eq(type, T_CIB_NOTIFY)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	free_xml(msg);
	return 0;
	}

	bool
	cib_native_dispatch(cib_t * cib)
	{
	gboolean stay_connected = TRUE;
	cib_native_opaque_t *native;

	if (cib == NULL) {
	crm_err("No CIB!");
	return FALSE;
	}

	crm_trace("dispatching %p", cib);
	native = cib->variant_opaque;
	while(crm_ipc_ready(native->ipc)) {

	if(crm_ipc_read(native->ipc) > 0) {
	const char *msg = crm_ipc_buffer(native->ipc);
	cib_native_dispatch_internal(msg, strlen(msg), cib);
	}

	if(crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static void
	cib_native_destroy(void *userdata)
	{
	cib_t *cib = userdata;
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_trace("destroying %p", userdata);
	cib->state = cib_disconnected;
	native->source = NULL;
	native->ipc = NULL;

	if(native->dnotify_fn) {
	native->dnotify_fn(userdata);
	}
	}

	int
	cib_native_signon_raw(cib_t * cib, const char name, enum cib_conn_type type, int async_fd)
	{
	int rc = pcmk_ok;
	const char *channel = NULL;
	cib_native_opaque_t *native = cib->variant_opaque;

	static struct ipc_client_callbacks cib_callbacks =
	{
	.dispatch = cib_native_dispatch_internal,
	.destroy = cib_native_destroy
	};

	cib->call_timeout = MAX_IPC_DELAY;

	if (type == cib_command) {
	cib->state = cib_connected_command;
	channel = cib_channel_rw;

	} else if (type == cib_command_nonblocking) {
	cib->state = cib_connected_command;
	channel = cib_channel_shm;

	} else if (type == cib_query) {
	cib->state = cib_connected_query;
	channel = cib_channel_ro;

	} else {
	return -ENOTCONN;
	}

	crm_trace("Connecting %s channel", channel);

	if (async_fd != NULL) {
	native->ipc = crm_ipc_new(channel, 0);

	if(native->ipc && crm_ipc_connect(native->ipc)) {
	*async_fd = crm_ipc_get_fd(native->ipc);

	} else if(native->ipc) {
	rc = -ENOTCONN;
	}

	} else {
	- native->source = mainloop_add_ipc_client(channel, 5121024 / 512k */, cib, &cib_callbacks);
	+ native->source = mainloop_add_ipc_client(channel, G_PRIORITY_HIGH, 5121024 / 512k */, cib, &cib_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (rc != pcmk_ok \|\| native->ipc == NULL \|\| crm_ipc_connected(native->ipc) == FALSE) {
	crm_debug("Connection unsuccessful (%d %p)", rc, native->ipc);
	rc = -ENOTCONN;
	}

	if (rc == pcmk_ok) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "stonith_command");

	crm_xml_add(hello, F_TYPE, T_CIB);
	crm_xml_add(hello, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	crm_xml_add_int(hello, F_CIB_CALLOPTS, cib_sync_call);

	if (crm_ipc_send(native->ipc, hello, &reply, -1) > 0) {
	const char *msg_type = crm_element_value(reply, F_CIB_OPERATION);

	rc = pcmk_ok;
	crm_log_xml_trace(reply, "reg-reply");

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else {
	native->token = crm_element_value_copy(reply, F_CIB_CLIENTID);
	if (native->token == NULL) {
	rc = -EPROTO;
	}
	}

	} else {
	rc = -ECOMM;
	}

	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	crm_debug("Connection to CIB successful");
	return pcmk_ok;
	}

	crm_debug("Connection to CIB failed: %s", pcmk_strerror(rc));
	cib_native_signoff(cib);
	return rc;
	}

	int
	cib_native_signoff(cib_t * cib)
	{
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_debug("Signing out of the CIB Service");

	if (native->ipc != NULL) {
	/* If attached to mainloop and it is active, _close() will result in:
	* - the source being removed from mainloop
	* - the dnotify callback being invoked
	* Otherwise, we are at least correctly disconnecting IPC
	*/
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	cib->state = cib_disconnected;
	cib->type = cib_none;

	return pcmk_ok;
	}

	int
	cib_native_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	if (cib->state != cib_disconnected) {
	rc = cib_native_signoff(cib);
	}

	if (cib->state == cib_disconnected) {
	cib_native_opaque_t *native = cib->variant_opaque;

	free(native->token);
	free(cib->variant_opaque);
	free(cib->cmds);
	free(cib);
	}

	return rc;
	}

	int
	cib_native_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options)
	{
	return cib_native_perform_op_delegate(cib, op, host, section,
	data, output_data, call_options, NULL);
	}

	int
	cib_native_perform_op_delegate(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options,
	const char *user_name)
	{
	int rc = pcmk_ok;
	int reply_id = 0;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the cib_errors enum
	* use 2 because we use it as (cib->call_id - 1) below
	*/
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);
	op_msg =
	cib_create_op(cib->call_id, native->token, op, host, section, data, call_options,
	user_name);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to CIB service (timeout=%ds)", op, cib->call_timeout);
	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, cib->call_timeout * 1000);
	free_xml(op_msg);

	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, cib->call_timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & cib_sync_call)) {
	crm_trace("Async call, returning %d", cib->call_id);
	CRM_CHECK(cib->call_id != 0, return -ENOMSG);
	free_xml(op_reply);
	return cib->call_id;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);
	if (reply_id == cib->call_id) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	crm_trace("Syncronous reply %d received", reply_id);
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (output_data == NULL \|\| (call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");

	} else if (tmp != NULL) {
	*output_data = copy_xml(tmp);
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	rc = -ENOMSG;
	goto done;

	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, cib->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	rc = -ENOMSG;
	goto done;
	}

	if (op_reply == NULL && cib->state == cib_disconnected) {
	rc = -ENOTCONN;

	} else if (rc == pcmk_ok && op_reply == NULL) {
	rc = -ETIME;
	}

	switch (rc) {
	case pcmk_ok:
	case -EPERM:
	break;

	/* This is an internal value that clients do not and should not care about */
	case -pcmk_err_diff_resync:
	rc = pcmk_ok;
	break;

	/* These indicate internal problems */
	case -EPROTO:
	case -ENOMSG:
	crm_err("Call failed: %s", pcmk_strerror(rc));
	if (op_reply) {
	crm_log_xml_err(op_reply, "Invalid reply");
	}
	break;

	default:
	if (safe_str_neq(op, CIB_OP_QUERY)) {
	crm_warn("Call failed: %s", pcmk_strerror(rc));
	}
	}

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("CIB disconnected");
	cib->state = cib_disconnected;
	}

	free_xml(op_reply);
	return rc;
	}

	int
	cib_native_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	cib_native_opaque_t *native = NULL;

	if (cib == NULL) {
	crm_err("No CIB!");
	return FALSE;
	}

	native = cib->variant_opaque;
	native->dnotify_fn = dnotify;

	return pcmk_ok;
	}

	int
	cib_native_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	int rc = pcmk_ok;
	xmlNode *notify_msg = create_xml_node(NULL, "cib-callback");
	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state != cib_disconnected) {
	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	rc = crm_ipc_send(native->ipc, notify_msg, NULL, 1000 * cib->call_timeout);
	if(rc <= 0) {
	crm_trace("Notification not registered: %d", rc);
	rc = -ECOMM;
	}
	}

	free_xml(notify_msg);
	return rc;
	}
	diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
	index 44c14c916f..d0fb696d8d 100644
	--- a/lib/cib/cib_remote.c
	+++ b/lib/cib/cib_remote.c
	@@ -1,641 +1,641 @@
	/*
	* Copyright (c) 2008 Andrew Beekhof
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <netdb.h>
	#include <termios.h>
	#include <sys/socket.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/mainloop.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	extern gnutls_anon_client_credentials anon_cred_c;
	extern gnutls_session *create_tls_session(int csock, int type);

	const int kx_prio[] = {
	GNUTLS_KX_ANON_DH,
	0
	};

	#else
	typedef void gnutls_session;
	#endif

	#include <arpa/inet.h>
	#include <sgtty.h>

	#define DH_BITS 1024

	struct remote_connection_s {
	int socket;
	gboolean encrypted;
	gnutls_session *session;
	mainloop_io_t *source;
	char *token;
	};

	typedef struct cib_remote_opaque_s {
	int flags;
	int socket;
	int port;
	char *server;
	char *user;
	char *passwd;
	struct remote_connection_s command;
	struct remote_connection_s callback;

	} cib_remote_opaque_t;

	void cib_remote_connection_destroy(gpointer user_data);
	int cib_remote_dispatch(gpointer user_data);
	int cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type);
	int cib_remote_signoff(cib_t * cib);
	int cib_remote_free(cib_t * cib);

	int cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options, const char *name);

	static int
	cib_remote_inputfd(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	return private->callback.socket;
	}

	static int
	cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
	{
	return -EPROTONOSUPPORT;
	}

	static int
	cib_remote_register_notification(cib_t * cib, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, "cib_command");
	cib_remote_opaque_t *private = cib->variant_opaque;

	crm_xml_add(notify_msg, F_CIB_OPERATION, T_CIB_NOTIFY);
	crm_xml_add(notify_msg, F_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, F_CIB_NOTIFY_ACTIVATE, enabled);
	crm_send_remote_msg(private->callback.session, notify_msg, private->callback.encrypted);
	free_xml(notify_msg);
	return pcmk_ok;
	}

	cib_t *
	cib_remote_new(const char server, const char user, const char *passwd, int port,
	gboolean encrypted)
	{
	cib_remote_opaque_t *private = NULL;
	cib_t *cib = cib_new_variant();

	private = calloc(1, sizeof(cib_remote_opaque_t));

	cib->variant = cib_remote;
	cib->variant_opaque = private;

	if (server) {
	private->server = strdup(server);
	}

	if (user) {
	private->user = strdup(user);
	}

	if (passwd) {
	private->passwd = strdup(passwd);
	}

	private->port = port;
	private->command.encrypted = encrypted;
	private->callback.encrypted = encrypted;

	/* assign variant specific ops */
	cib->delegate_fn = cib_remote_perform_op;
	cib->cmds->signon = cib_remote_signon;
	cib->cmds->signoff = cib_remote_signoff;
	cib->cmds->free = cib_remote_free;
	cib->cmds->inputfd = cib_remote_inputfd;

	cib->cmds->register_notification = cib_remote_register_notification;
	cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify;

	return cib;
	}

	static int
	cib_tls_close(cib_t * cib)
	{
	cib_remote_opaque_t *private = cib->variant_opaque;

	shutdown(private->command.socket, SHUT_RDWR); /* no more receptions */
	shutdown(private->callback.socket, SHUT_RDWR); /* no more receptions */
	close(private->command.socket);
	close(private->callback.socket);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (private->command.encrypted) {
	gnutls_bye(*(private->command.session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->command.session));
	gnutls_free(private->command.session);

	gnutls_bye(*(private->callback.session), GNUTLS_SHUT_RDWR);
	gnutls_deinit(*(private->callback.session));
	gnutls_free(private->callback.session);

	gnutls_anon_free_client_credentials(anon_cred_c);
	gnutls_global_deinit();
	}
	#endif
	return 0;
	}

	static int
	cib_tls_signon(cib_t * cib, struct remote_connection_s *connection)
	{
	int sock;
	cib_remote_opaque_t *private = cib->variant_opaque;
	struct sockaddr_in addr;
	int rc = 0;
	char *server = private->server;

	int ret_ga;
	struct addrinfo *res;
	struct addrinfo hints;

	xmlNode *answer = NULL;
	xmlNode *login = NULL;

	static struct mainloop_fd_callbacks cib_fd_callbacks =
	{
	.dispatch = cib_remote_dispatch,
	.destroy = cib_remote_connection_destroy,
	};

	connection->socket = 0;
	connection->session = NULL;

	/* create socket */
	sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
	if (sock == -1) {
	crm_perror(LOG_ERR, "Socket creation failed");
	return -1;
	}

	/* getaddrinfo */
	bzero(&hints, sizeof(struct addrinfo));
	hints.ai_flags = AI_CANONNAME;
	hints.ai_family = AF_INET;
	hints.ai_socktype = SOCK_RAW;

	if (hints.ai_family == AF_INET6) {
	hints.ai_protocol = IPPROTO_ICMPV6;
	} else {
	hints.ai_protocol = IPPROTO_ICMP;
	}

	crm_debug("Looking up %s", server);
	ret_ga = getaddrinfo(server, NULL, &hints, &res);
	if (ret_ga) {
	crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
	close(sock);
	return -1;
	}

	if (res->ai_canonname) {
	server = res->ai_canonname;
	}

	crm_debug("Got address %s for %s", server, private->server);

	if (!res->ai_addr) {
	fprintf(stderr, "getaddrinfo failed");
	exit(1);
	}
	#if 1
	memcpy(&addr, res->ai_addr, res->ai_addrlen);
	#else
	/* connect to server */
	memset(&addr, 0, sizeof(addr));
	addr.sin_family = AF_INET;
	addr.sin_addr.s_addr = inet_addr(server);
	#endif
	addr.sin_port = htons(private->port);

	if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
	crm_perror(LOG_ERR, "Connection to %s:%d failed", server, private->port);
	close(sock);
	return -1;
	}

	if (connection->encrypted) {
	/* initialize GnuTls lib */
	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_global_init();
	gnutls_anon_allocate_client_credentials(&anon_cred_c);

	/* bind the socket to GnuTls lib */
	connection->session = create_tls_session(sock, GNUTLS_CLIENT);
	if (connection->session == NULL) {
	crm_perror(LOG_ERR, "Session creation for %s:%d failed", server, private->port);
	close(sock);
	cib_tls_close(cib);
	return -1;
	}
	#else
	return -EPROTONOSUPPORT;
	#endif
	} else {
	connection->session = GUINT_TO_POINTER(sock);
	}

	/* login to server */
	login = create_xml_node(NULL, "cib_command");
	crm_xml_add(login, "op", "authenticate");
	crm_xml_add(login, "user", private->user);
	crm_xml_add(login, "password", private->passwd);
	crm_xml_add(login, "hidden", "password");

	crm_send_remote_msg(connection->session, login, connection->encrypted);
	free_xml(login);

	answer = crm_recv_remote_msg(connection->session, connection->encrypted);
	crm_log_xml_trace(answer, "Reply");
	if (answer == NULL) {
	rc = -EPROTO;

	} else {
	/* grab the token */
	const char *msg_type = crm_element_value(answer, F_CIB_OPERATION);
	const char *tmp_ticket = crm_element_value(answer, F_CIB_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	rc = -EPROTO;

	} else {
	connection->token = strdup(tmp_ticket);
	}
	}

	if (rc != 0) {
	cib_tls_close(cib);
	}

	connection->socket = sock;
	- connection->source = mainloop_add_fd("cib-remote", connection->socket, cib, &cib_fd_callbacks);
	+ connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH, connection->socket, cib, &cib_fd_callbacks);
	return rc;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(user_data);
	#endif
	return;
	}

	int
	cib_remote_dispatch(gpointer user_data)
	{
	cib_t *cib = user_data;
	cib_remote_opaque_t *private = cib->variant_opaque;

	xmlNode *msg = NULL;
	const char *type = NULL;

	crm_info("Message on callback channel");
	msg = crm_recv_remote_msg(private->callback.session, private->callback.encrypted);

	type = crm_element_value(msg, F_TYPE);
	crm_trace("Activating %s callbacks...", type);

	if (safe_str_eq(type, T_CIB)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (safe_str_eq(type, T_CIB_NOTIFY)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	if (msg != NULL) {
	free_xml(msg);
	return 0;
	}
	return -1;
	}

	int
	cib_remote_signon(cib_t * cib, const char *name, enum cib_conn_type type)
	{
	int rc = pcmk_ok;
	cib_remote_opaque_t *private = cib->variant_opaque;

	if (private->passwd == NULL) {
	struct termios settings;
	int rc;

	rc = tcgetattr(0, &settings);
	settings.c_lflag &= ~ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);

	fprintf(stderr, "Password: ");
	private->passwd = calloc(1, 1024);
	rc = scanf("%s", private->passwd);
	fprintf(stdout, "\n");
	/* fprintf(stderr, "entered: '%s'\n", buffer); */
	if (rc < 1) {
	private->passwd = NULL;
	}

	settings.c_lflag \|= ECHO;
	rc = tcsetattr(0, TCSANOW, &settings);
	}

	if (private->server == NULL \|\| private->user == NULL) {
	rc = -EINVAL;
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->command));
	}

	if (rc == pcmk_ok) {
	rc = cib_tls_signon(cib, &(private->callback));
	}

	if (rc == pcmk_ok) {
	xmlNode *hello =
	cib_create_op(0, private->callback.token, CRM_OP_REGISTER, NULL, NULL, NULL, 0, NULL);
	crm_xml_add(hello, F_CIB_CLIENTNAME, name);
	crm_send_remote_msg(private->command.session, hello, private->command.encrypted);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	fprintf(stderr, "%s: Opened connection to %s:%d\n", name, private->server, private->port);
	cib->state = cib_connected_command;
	cib->type = cib_command;

	} else {
	fprintf(stderr, "%s: Connection to %s:%d failed: %s\n",
	name, private->server, private->port, pcmk_strerror(rc));
	}

	return rc;
	}

	int
	cib_remote_signoff(cib_t * cib)
	{
	int rc = pcmk_ok;

	/* cib_remote_opaque_t private = cib->variant_opaque; /

	crm_debug("Signing out of the CIB Service");
	#ifdef HAVE_GNUTLS_GNUTLS_H
	cib_tls_close(cib);
	#endif

	cib->state = cib_disconnected;
	cib->type = cib_none;

	return rc;
	}

	int
	cib_remote_free(cib_t * cib)
	{
	int rc = pcmk_ok;

	crm_warn("Freeing CIB");
	if (cib->state != cib_disconnected) {
	rc = cib_remote_signoff(cib);
	if (rc == pcmk_ok) {
	cib_remote_opaque_t *private = cib->variant_opaque;

	free(private->server);
	free(private->user);
	free(private->passwd);
	free(cib->cmds);
	free(private);
	free(cib);
	}
	}

	return rc;
	}

	static gboolean timer_expired = FALSE;
	static struct timer_rec_s *sync_timer = NULL;
	static gboolean
	cib_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	timer_expired = TRUE;
	crm_err("Call %d timed out after %ds", timer->call_id, timer->timeout);

	/* Always return TRUE, never remove the handler
	* We do that after the while-loop in cib_native_perform_op()
	*/
	return TRUE;
	}

	int
	cib_remote_perform_op(cib_t * cib, const char op, const char host, const char *section,
	xmlNode * data, xmlNode ** output_data, int call_options, const char *name)
	{
	int rc = pcmk_ok;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_remote_opaque_t *private = cib->variant_opaque;

	if (sync_timer == NULL) {
	sync_timer = calloc(1, sizeof(struct timer_rec_s));
	}

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	cib->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the cib_errors enum
	* use 2 because we use it as (cib->call_id - 1) below
	*/
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	op_msg =
	cib_create_op(cib->call_id, private->callback.token, op, host, section, data, call_options,
	NULL);
	if (op_msg == NULL) {
	return -EPROTO;
	}

	crm_trace("Sending %s message to CIB service", op);
	crm_send_remote_msg(private->command.session, op_msg, private->command.encrypted);
	free_xml(op_msg);

	if ((call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");
	return pcmk_ok;

	} else if (!(call_options & cib_sync_call)) {
	return cib->call_id;
	}

	crm_trace("Waiting for a syncronous reply");

	if (cib->call_timeout > 0) {
	/* We need this, even with msgfromIPC_timeout(), because we might
	* get other/older replies that don't match the active request
	*/
	timer_expired = FALSE;
	sync_timer->call_id = cib->call_id;
	sync_timer->timeout = cib->call_timeout * 1000;
	sync_timer->ref = g_timeout_add(sync_timer->timeout, cib_timeout_handler, sync_timer);
	}

	while (timer_expired == FALSE) {
	int reply_id = -1;
	int msg_id = cib->call_id;

	op_reply = crm_recv_remote_msg(private->command.session, private->command.encrypted);
	if (op_reply == NULL) {
	break;
	}

	crm_element_value_int(op_reply, F_CIB_CALLID, &reply_id);
	CRM_CHECK(reply_id > 0, free_xml(op_reply);
	if (sync_timer->ref > 0) {
	g_source_remove(sync_timer->ref); sync_timer->ref = 0;}
	return -ENOMSG) ;

	if (reply_id == msg_id) {
	break;

	} else if (reply_id < msg_id) {
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");

	} else if ((reply_id - 10000) > msg_id) {
	/* wrap-around case */
	crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
	crm_log_xml_trace(op_reply, "Old reply");
	} else {
	crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
	}

	free_xml(op_reply);
	op_reply = NULL;
	}

	if (sync_timer->ref > 0) {
	g_source_remove(sync_timer->ref);
	sync_timer->ref = 0;
	}

	if (timer_expired) {
	return -ETIME;
	}

	/* if(IPC_ISRCONN(native->command_channel) == FALSE) { */
	/* crm_err("CIB disconnected: %d", */
	/* native->command_channel->ch_status); */
	/* cib->state = cib_disconnected; */
	/* } */

	if (op_reply == NULL) {
	crm_err("No reply message - empty");
	return -ENOMSG;
	}

	crm_trace("Syncronous reply received");

	/* Start processing the reply... */
	if (crm_element_value_int(op_reply, F_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok \|\| rc == -EPERM) {
	crm_log_xml_debug(op_reply, "passed");

	} else {
	/* } else if(rc == -ETIME) { */
	crm_err("Call failed: %s", pcmk_strerror(rc));
	crm_log_xml_warn(op_reply, "failed");
	}

	if (output_data == NULL) {
	/* do nothing more */

	} else if (!(call_options & cib_discard_reply)) {
	xmlNode *tmp = get_message_xml(op_reply, F_CIB_CALLDATA);

	if (tmp == NULL) {
	crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1);
	} else {
	*output_data = copy_xml(tmp);
	}
	}

	free_xml(op_reply);

	return rc;
	}
	diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c
	index be69bc4e82..383d923d0d 100644
	--- a/lib/cluster/corosync.c
	+++ b/lib/cluster/corosync.c
	@@ -1,1092 +1,1092 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <bzlib.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>
	#include <sys/utsname.h>

	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/cpg.h>
	#include <corosync/cfg.h>
	#include <corosync/cmap.h>
	#include <corosync/quorum.h>

	#include <crm/msg_xml.h>

	cpg_handle_t pcmk_cpg_handle = 0;

	struct cpg_name pcmk_cpg_group = {
	.length = 0,
	.value[0] = 0,
	};

	quorum_handle_t pcmk_quorum_handle = 0;
	gboolean(*quorum_app_callback) (unsigned long long seq, gboolean quorate) = NULL;

	static char *pcmk_uname = NULL;
	static int pcmk_uname_len = 0;
	static uint32_t pcmk_nodeid = 0;

	#define cs_repeat(counter, max, code) do { \
	code; \
	if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	counter++; \
	crm_debug("Retrying operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while(counter < max)

	#ifndef INTERFACE_MAX
	# define INTERFACE_MAX 2 /* from the private coroapi.h header */
	#endif

	static gboolean
	corosync_name_is_valid(const char key, const char name)
	{
	int octet;

	if(name == NULL) {
	crm_trace("%s is empty", key);
	return FALSE;

	} else if(sscanf(name, "%d.%d.%d.%d", &octet, &octet, &octet, &octet) == 4) {
	crm_trace("%s contains an ipv4 address, ignoring: %s", key, name);
	return FALSE;

	} else if(strstr(name, ":") != NULL) {
	crm_trace("%s contains an ipv4 address, ignoring: %s", key, name);
	return FALSE;
	}
	crm_trace("%s is valid", key);
	return TRUE;
	}

	/*
	* CFG functionality stolen from node_name() in corosync-quorumtool.c
	* This resolves the first address assigned to a node and returns the name or IP address.
	*/
	static char *corosync_node_name(cmap_handle_t cmap_handle, uint32_t nodeid)
	{
	int lpc = 0;
	int rc = CS_OK;
	int retries = 0;
	char *name = NULL;

	cmap_handle_t local_handle = 0;
	corosync_cfg_handle_t cfg_handle = 0;
	static corosync_cfg_callbacks_t cfg_callbacks = {};

	if(cmap_handle == 0 && local_handle == 0) {
	retries = 0;
	crm_trace("Initializing CMAP connection");
	do {
	rc = cmap_initialize(&local_handle);
	if(rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc), retries);
	sleep(retries);
	}

	} while(retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", cs_strerror(rc));
	local_handle = 0;
	}
	}

	if(cmap_handle == 0) {
	cmap_handle = local_handle;
	}

	while(name == NULL && cmap_handle != 0) {
	uint32_t id = 0;
	char *key = NULL;

	key = g_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &id);
	crm_trace("Checking %u vs %u from %s", nodeid, id, key);
	g_free(key);

	if(rc != CS_OK) {
	break;
	}

	if(nodeid == id) {
	crm_trace("Searching for node name for %u in nodelist.node.%d %s", nodeid, lpc, name);
	if(name == NULL) {
	key = g_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
	rc = cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s", key, name);

	if(corosync_name_is_valid(key, name) == FALSE) {
	free(name); name = NULL;
	}
	g_free(key);
	}

	if(name == NULL) {
	key = g_strdup_printf("nodelist.node.%d.name", lpc);
	rc = cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s %d", key, name, rc);

	if(corosync_name_is_valid(key, name) == FALSE) {
	free(name); name = NULL;
	}
	g_free(key);
	}
	break;
	}

	lpc++;
	}

	if(name == NULL) {
	retries = 0;
	crm_trace("Initializing CFG connection");
	do {
	rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks);
	if(rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc), retries);
	sleep(retries);
	}

	} while(retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to the Corosync CFG API, error %d", cs_strerror(rc));
	cfg_handle = 0;
	}
	}

	if(name == NULL && cfg_handle != 0) {
	int numaddrs;
	char buf[INET6_ADDRSTRLEN];

	socklen_t addrlen;
	struct sockaddr_storage *ss;
	corosync_cfg_node_address_t addrs[INTERFACE_MAX];

	rc = corosync_cfg_get_node_addrs(cfg_handle, nodeid, INTERFACE_MAX, &numaddrs, addrs);
	if (rc == CS_OK) {
	ss = (struct sockaddr_storage *)addrs[0].address;
	if (ss->ss_family == AF_INET6) {
	addrlen = sizeof(struct sockaddr_in6);
	} else {
	addrlen = sizeof(struct sockaddr_in);
	}

	if (getnameinfo((struct sockaddr *)addrs[0].address, addrlen, buf, sizeof(buf), NULL, 0, 0) == 0) {
	crm_notice("Inferred node name '%s' for nodeid %u from DNS", buf, nodeid);

	if(corosync_name_is_valid("DNS", buf)) {
	name = strdup(buf);
	}
	}
	} else {
	crm_debug("Unable to get node address for nodeid %u: %s", nodeid, cs_strerror(rc));
	}
	cmap_finalize(cfg_handle);
	}

	if(local_handle) {
	cmap_finalize(local_handle);
	}

	if(name == NULL) {
	crm_err("Unable to get node name for nodeid %u", nodeid);
	}
	return name;
	}

	enum crm_ais_msg_types
	text2msg_type(const char *text)
	{
	int type = crm_msg_none;

	CRM_CHECK(text != NULL, return type);
	if (safe_str_eq(text, "ais")) {
	type = crm_msg_ais;
	} else if (safe_str_eq(text, "crm_plugin")) {
	type = crm_msg_ais;
	} else if (safe_str_eq(text, CRM_SYSTEM_CIB)) {
	type = crm_msg_cib;
	} else if (safe_str_eq(text, CRM_SYSTEM_CRMD)) {
	type = crm_msg_crmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_DC)) {
	type = crm_msg_crmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
	type = crm_msg_te;
	} else if (safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
	type = crm_msg_pe;
	} else if (safe_str_eq(text, CRM_SYSTEM_LRMD)) {
	type = crm_msg_lrmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
	type = crm_msg_stonithd;
	} else if (safe_str_eq(text, "stonith-ng")) {
	type = crm_msg_stonith_ng;
	} else if (safe_str_eq(text, "attrd")) {
	type = crm_msg_attrd;

	} else {
	/* This will normally be a transient client rather than
	* a cluster daemon. Set the type to the pid of the client
	*/
	int scan_rc = sscanf(text, "%d", &type);

	if (scan_rc != 1) {
	/* Ensure its sane */
	type = crm_msg_none;
	}
	}
	return type;
	}

	static char *ais_cluster_name = NULL;

	gboolean
	crm_get_cluster_name(char **cname)
	{
	CRM_CHECK(cname != NULL, return FALSE);
	if (ais_cluster_name) {
	*cname = strdup(ais_cluster_name);
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	send_ais_text(int class, const char *data,
	gboolean local, const char *node, enum crm_ais_msg_types dest)
	{
	static int msg_id = 0;
	static int local_pid = 0;

	int retries = 0;
	int rc = CS_OK;
	int buf_len = sizeof(cs_ipc_header_response_t);

	char *buf = NULL;
	struct iovec iov;
	const char *transport = "pcmk";
	AIS_Message *ais_msg = NULL;
	enum crm_ais_msg_types sender = text2msg_type(crm_system_name);

	/* There are only 6 handlers registered to crm_lib_service in plugin.c */
	CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class); return FALSE);

	if (data == NULL) {
	data = "";
	}

	if (local_pid == 0) {
	local_pid = getpid();
	}

	if (sender == crm_msg_none) {
	sender = local_pid;
	}

	ais_msg = calloc(1, sizeof(AIS_Message));

	ais_msg->id = msg_id++;
	ais_msg->header.id = class;
	ais_msg->header.error = CS_OK;

	ais_msg->host.type = dest;
	ais_msg->host.local = local;
	if (node) {
	ais_msg->host.size = strlen(node);
	memset(ais_msg->host.uname, 0, MAX_NAME);
	memcpy(ais_msg->host.uname, node, ais_msg->host.size);
	ais_msg->host.id = 0;

	} else {
	ais_msg->host.size = 0;
	memset(ais_msg->host.uname, 0, MAX_NAME);
	ais_msg->host.id = 0;
	}

	ais_msg->sender.id = 0;
	ais_msg->sender.type = sender;
	ais_msg->sender.pid = local_pid;
	ais_msg->sender.size = pcmk_uname_len;
	memset(ais_msg->sender.uname, 0, MAX_NAME);
	memcpy(ais_msg->sender.uname, pcmk_uname, ais_msg->sender.size);

	ais_msg->size = 1 + strlen(data);

	if (ais_msg->size < CRM_BZ2_THRESHOLD) {
	failback:
	ais_msg = realloc(ais_msg, sizeof(AIS_Message) + ais_msg->size);
	memcpy(ais_msg->data, data, ais_msg->size);

	} else {
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	unsigned int len = (ais_msg->size * 1.1) + 600; /* recomended size */

	crm_trace("Compressing message payload");
	compressed = malloc( len);

	rc = BZ2_bzBuffToBuffCompress(compressed, &len, uncompressed, ais_msg->size, CRM_BZ2_BLOCKS,
	0, CRM_BZ2_WORK);

	free(uncompressed);

	if (rc != BZ_OK) {
	crm_err("Compression failed: %d", rc);
	free(compressed);
	goto failback;
	}

	ais_msg = realloc(ais_msg, sizeof(AIS_Message) + len + 1);
	memcpy(ais_msg->data, compressed, len);
	ais_msg->data[len] = 0;
	free(compressed);

	ais_msg->is_compressed = TRUE;
	ais_msg->compressed_size = len;

	crm_trace("Compression details: %d -> %d", ais_msg->size, ais_data_len(ais_msg));
	}

	ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg);

	crm_trace("Sending%s message %d to %s.%s (data=%d, total=%d)",
	ais_msg->is_compressed ? " compressed" : "",
	ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(dest),
	ais_data_len(ais_msg), ais_msg->header.size);

	iov.iov_base = ais_msg;
	iov.iov_len = ais_msg->header.size;
	buf = realloc(buf, buf_len);

	do {
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	retries++;
	crm_info("Peer overloaded or membership in flux:"
	" Re-sending message (Attempt %d of 20)", retries);
	sleep(retries); /* Proportional back off */
	}

	errno = 0;
	transport = "cpg";
	CRM_CHECK(dest != crm_msg_ais, rc = CS_ERR_MESSAGE_ERROR; goto bail);
	rc = cpg_mcast_joined(pcmk_cpg_handle, CPG_TYPE_AGREED, &iov, 1);
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED;
	int rc2 = cpg_flow_control_state_get(pcmk_cpg_handle, &fc_state);

	if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
	crm_warn("Connection overloaded, cannot send messages");
	goto bail;

	} else if (rc2 != CS_OK) {
	crm_warn("Could not determin the connection state: %s (%d)",
	ais_error2text(rc2), rc2);
	goto bail;
	}
	}

	} while ((rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) && retries < 20);

	bail:
	if (rc != CS_OK) {
	crm_perror(LOG_ERR, "Sending message %d via %s: FAILED (rc=%d): %s",
	ais_msg->id, transport, rc, ais_error2text(rc));

	} else {
	crm_trace("Message %d: sent", ais_msg->id);
	}

	free(buf);
	free(ais_msg);
	return (rc == CS_OK);
	}

	gboolean
	send_ais_message(xmlNode * msg, gboolean local, const char *node, enum crm_ais_msg_types dest)
	{
	gboolean rc = TRUE;
	char *data = dump_xml_unformatted(msg);
	rc = send_ais_text(crm_class_cluster, data, local, node, dest);
	free(data);
	return rc;
	}

	void
	terminate_ais_connection(void)
	{
	crm_notice("Disconnecting from Corosync");

	if(pcmk_cpg_handle) {
	crm_trace("Disconnecting CPG");
	cpg_leave(pcmk_cpg_handle, &pcmk_cpg_group);
	cpg_finalize(pcmk_cpg_handle);
	pcmk_cpg_handle = 0;

	} else {
	crm_info("No CPG connection");
	}

	if(pcmk_quorum_handle) {
	crm_trace("Disconnecting quorum");
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;

	} else {
	crm_info("No Quorum connection");
	}
	}

	int ais_membership_timer = 0;
	gboolean ais_membership_force = FALSE;

	static gboolean
	ais_dispatch_message(AIS_Message * msg, gboolean(dispatch) (AIS_Message , char *, int))
	{
	char *data = NULL;
	char *uncompressed = NULL;

	xmlNode *xml = NULL;

	CRM_ASSERT(msg != NULL);

	crm_trace("Got new%s message (size=%d, %d, %d)",
	msg->is_compressed ? " compressed" : "",
	ais_data_len(msg), msg->size, msg->compressed_size);

	data = msg->data;
	if (msg->is_compressed && msg->size > 0) {
	int rc = BZ_OK;
	unsigned int new_size = msg->size + 1;

	if (check_message_sanity(msg, NULL) == FALSE) {
	goto badmsg;
	}

	crm_trace("Decompressing message data");
	uncompressed = calloc(1, new_size);
	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, data, msg->compressed_size, 1, 0);

	if (rc != BZ_OK) {
	crm_err("Decompression failed: %d", rc);
	goto badmsg;
	}

	CRM_ASSERT(rc == BZ_OK);
	CRM_ASSERT(new_size == msg->size);

	data = uncompressed;

	} else if (check_message_sanity(msg, data) == FALSE) {
	goto badmsg;

	} else if (safe_str_eq("identify", data)) {
	int pid = getpid();
	char *pid_s = crm_itoa(pid);

	send_ais_text(crm_class_cluster, pid_s, TRUE, NULL, crm_msg_ais);
	free(pid_s);
	goto done;
	}

	if (msg->header.id != crm_class_members) {
	/* Is this even needed anymore? */
	crm_get_peer(msg->sender.id, msg->sender.uname);
	}

	if (msg->header.id == crm_class_rmpeer) {
	uint32_t id = crm_int_helper(data, NULL);

	crm_info("Removing peer %s/%u", data, id);
	reap_crm_member(id);
	goto done;
	}

	crm_trace("Payload: %s", data);
	if (dispatch != NULL) {
	dispatch(msg, data, 0);
	}

	done:
	free(uncompressed);
	free_xml(xml);
	return TRUE;

	badmsg:
	crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
	" min=%d, total=%d, size=%d, bz2_size=%d",
	msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	msg->sender.pid, (int)sizeof(AIS_Message),
	msg->header.size, msg->size, msg->compressed_size);
	goto done;
	}

	gboolean(pcmk_cpg_dispatch_fn) (AIS_Message , char *, int) = NULL;

	static int
	pcmk_cpg_dispatch(gpointer user_data)
	{
	int rc = 0;

	pcmk_cpg_dispatch_fn = user_data;
	rc = cpg_dispatch(pcmk_cpg_handle, CS_DISPATCH_ALL);
	if (rc != CS_OK) {
	crm_err("Connection to the CPG API failed: %d", rc);
	return -1;
	}
	return 0;
	}

	static void
	pcmk_cpg_deliver(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	AIS_Message ais_msg = (AIS_Message ) msg;

	if (ais_msg->sender.id > 0 && ais_msg->sender.id != nodeid) {
	crm_err("Nodeid mismatch from %d.%d: claimed nodeid=%u", nodeid, pid, ais_msg->sender.id);
	return;

	} else if (ais_msg->host.size != 0 && safe_str_neq(ais_msg->host.uname, pcmk_uname)) {
	/* Not for us */
	return;
	}

	ais_msg->sender.id = nodeid;
	if (ais_msg->sender.size == 0) {
	crm_node_t *peer = crm_get_peer(nodeid, NULL);

	if (peer == NULL) {
	crm_err("Peer with nodeid=%u is unknown", nodeid);

	} else if (peer->uname == NULL) {
	crm_err("No uname for peer with nodeid=%u", nodeid);

	} else {
	crm_notice("Fixing uname for peer with nodeid=%u", nodeid);
	ais_msg->sender.size = strlen(peer->uname);
	memset(ais_msg->sender.uname, 0, MAX_NAME);
	memcpy(ais_msg->sender.uname, peer->uname, ais_msg->sender.size);
	}
	}

	ais_dispatch_message(ais_msg, pcmk_cpg_dispatch_fn);
	}

	static void
	pcmk_cpg_membership(cpg_handle_t handle,
	const struct cpg_name *groupName,
	const struct cpg_address *member_list, size_t member_list_entries,
	const struct cpg_address *left_list, size_t left_list_entries,
	const struct cpg_address *joined_list, size_t joined_list_entries)
	{
	int i;
	gboolean found = FALSE;
	static int counter = 0;

	for (i = 0; i < left_list_entries; i++) {
	crm_node_t *peer = crm_get_peer(left_list[i].nodeid, NULL);
	crm_info("Left[%d.%d] %s.%d ", counter, i, groupName->value, left_list[i].nodeid);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, OFFLINESTATUS);
	}

	for (i = 0; i < joined_list_entries; i++) {
	crm_info("Joined[%d.%d] %s.%d ", counter, i, groupName->value, joined_list[i].nodeid);
	}

	for (i = 0; i < member_list_entries; i++) {
	crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL);
	crm_info("Member[%d.%d] %s.%d ", counter, i, groupName->value, member_list[i].nodeid);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
	if(pcmk_nodeid == member_list[i].nodeid) {
	found = TRUE;
	}
	}

	if(!found) {
	crm_err("We're not part of CPG group %s anymore!", groupName->value);
	/* Possibly re-call cpg_join() */
	}

	counter++;
	}

	cpg_callbacks_t cpg_callbacks = {
	.cpg_deliver_fn = pcmk_cpg_deliver,
	.cpg_confchg_fn = pcmk_cpg_membership,
	};

	static gboolean
	init_cpg_connection(gboolean(dispatch) (AIS_Message , char , int), void (destroy) (gpointer),
	uint32_t * nodeid)
	{
	int rc = -1;
	int fd = 0;
	int retries = 0;
	crm_node_t *peer = NULL;
	struct mainloop_fd_callbacks cpg_fd_callbacks = {
	.dispatch = pcmk_cpg_dispatch,
	.destroy = destroy,
	};

	strcpy(pcmk_cpg_group.value, crm_system_name);
	pcmk_cpg_group.length = strlen(crm_system_name) + 1;

	cs_repeat(retries, 30, rc = cpg_initialize(&pcmk_cpg_handle, &cpg_callbacks));
	if (rc != CS_OK) {
	crm_err("Could not connect to the Cluster Process Group API: %d\n", rc);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_local_get(pcmk_cpg_handle, (unsigned int *)nodeid));
	if (rc != CS_OK) {
	crm_err("Could not get local node id from the CPG API");
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_join(pcmk_cpg_handle, &pcmk_cpg_group));
	if (rc != CS_OK) {
	crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
	goto bail;
	}

	rc = cpg_fd_get(pcmk_cpg_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %d\n", rc);
	goto bail;
	}

	- mainloop_add_fd("corosync-cpg", fd, dispatch, &cpg_fd_callbacks);
	+ mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, dispatch, &cpg_fd_callbacks);

	bail:
	if (rc != CS_OK) {
	cpg_finalize(pcmk_cpg_handle);
	return FALSE;
	}

	peer = crm_get_peer(pcmk_nodeid, pcmk_uname);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
	return TRUE;
	}

	static int
	pcmk_quorum_dispatch(gpointer user_data)
	{
	int rc = 0;

	rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL);
	if (rc < 0) {
	crm_err("Connection to the Quorum API failed: %d", rc);
	return -1;
	}
	return 0;
	}

	static void
	corosync_mark_unseen_peer_dead(gpointer key, gpointer value, gpointer user_data)
	{
	int *seq = user_data;
	crm_node_t *node = value;

	if (node->last_seen != *seq && node->state && crm_str_eq(CRM_NODE_LOST, node->state, TRUE) == FALSE) {
	crm_notice("Node %d/%s was not seen in the previous transition", node->id, node->uname);
	crm_update_peer_state(__FUNCTION__, node, CRM_NODE_LOST, 0);
	}
	}

	static void
	corosync_mark_node_unseen(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;

	node->last_seen = 0;
	}

	static void
	pcmk_quorum_notification(quorum_handle_t handle,
	uint32_t quorate,
	uint64_t ring_id, uint32_t view_list_entries, uint32_t * view_list)
	{
	int i;
	static gboolean init_phase = TRUE;

	if (quorate != crm_have_quorum) {
	crm_notice("Membership " U64T ": quorum %s (%lu)", ring_id,
	quorate ? "acquired" : "lost", (long unsigned int)view_list_entries);
	crm_have_quorum = quorate;

	} else {
	crm_info("Membership " U64T ": quorum %s (%lu)", ring_id,
	quorate ? "retained" : "still lost", (long unsigned int)view_list_entries);
	}

	if(view_list_entries == 0 && init_phase) {
	crm_info("Corosync membership is still forming, ignoring");
	return;
	}

	init_phase = FALSE;
	g_hash_table_foreach(crm_peer_cache, corosync_mark_node_unseen, NULL);

	for (i = 0; i < view_list_entries; i++) {
	uint32_t id = view_list[i];
	char *name = NULL;
	crm_node_t *node = NULL;

	crm_debug("Member[%d] %d ", i, id);

	node = crm_get_peer(id, NULL);
	if(node->uname == NULL) {
	crm_info("Obtaining name for new node %u", id);
	name = corosync_node_name(0, id);
	node = crm_get_peer(id, name);
	}

	crm_update_peer_state(__FUNCTION__, node, CRM_NODE_MEMBER, ring_id);
	free(name);
	}

	crm_trace("Reaping unseen nodes...");
	g_hash_table_foreach(crm_peer_cache, corosync_mark_unseen_peer_dead, &ring_id);

	if (quorum_app_callback) {
	quorum_app_callback(ring_id, quorate);
	}
	}

	quorum_callbacks_t quorum_callbacks = {
	.quorum_notify_fn = pcmk_quorum_notification,
	};

	gboolean
	init_quorum_connection(gboolean(*dispatch) (unsigned long long, gboolean),
	void (*destroy) (gpointer))
	{
	int rc = -1;
	int fd = 0;
	int quorate = 0;
	uint32_t quorum_type = 0;
	struct mainloop_fd_callbacks quorum_fd_callbacks;
	quorum_fd_callbacks.dispatch = pcmk_quorum_dispatch;
	quorum_fd_callbacks.destroy = destroy;

	crm_debug("Configuring Pacemaker to obtain quorum from Corosync");

	rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks, &quorum_type);
	if (rc != CS_OK) {
	crm_err("Could not connect to the Quorum API: %d\n", rc);
	goto bail;

	} else if (quorum_type != QUORUM_SET) {
	crm_err("Corosync quorum is not configured\n");
	goto bail;
	}

	rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
	if (rc != CS_OK) {
	crm_err("Could not obtain the current Quorum API state: %d\n", rc);
	goto bail;
	}

	crm_notice("Quorum %s", quorate ? "acquired" : "lost");
	quorum_app_callback = dispatch;
	crm_have_quorum = quorate;

	rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES \| CS_TRACK_CURRENT);
	if (rc != CS_OK) {
	crm_err("Could not setup Quorum API notifications: %d\n", rc);
	goto bail;
	}

	rc = quorum_fd_get(pcmk_quorum_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the Quorum API connection: %d\n", rc);
	goto bail;
	}

	- mainloop_add_fd("quorum", fd, dispatch, &quorum_fd_callbacks);
	+ mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);

	corosync_initialize_nodelist(NULL, FALSE, NULL);

	bail:
	if (rc != CS_OK) {
	quorum_finalize(pcmk_quorum_handle);
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	init_ais_connection(gboolean(dispatch) (AIS_Message , char , int), void (destroy) (gpointer),
	char our_uuid, char our_uname, int *nodeid)
	{
	int retries = 0;

	while (retries++ < 30) {
	int rc = init_ais_connection_once(dispatch, destroy, our_uuid, our_uname, nodeid);

	switch (rc) {
	case CS_OK:
	return TRUE;
	break;
	case CS_ERR_TRY_AGAIN:
	case CS_ERR_QUEUE_FULL:
	break;
	default:
	return FALSE;
	}
	}

	crm_err("Retry count exceeded: %d", retries);
	return FALSE;
	}

	gboolean
	init_ais_connection_once(gboolean(dispatch) (AIS_Message , char *, int),
	void (destroy) (gpointer), char our_uuid, char our_uname, int nodeid)
	{
	struct utsname res;
	enum cluster_type_e stack = get_cluster_type();

	crm_peer_init();

	/* Here we just initialize comms */
	if(stack != pcmk_cluster_corosync) {
	crm_err("Invalid cluster type: %s (%d)", name_for_cluster_type(stack), stack);
	return FALSE;
	}

	if (init_cpg_connection(dispatch, destroy, &pcmk_nodeid) == FALSE) {
	return FALSE;
	} else if (uname(&res) < 0) {
	crm_perror(LOG_ERR, "Could not determin the current host");
	exit(100);

	} else {
	pcmk_uname = strdup(res.nodename);
	}

	crm_info("Connection to '%s': established", name_for_cluster_type(stack));

	CRM_ASSERT(pcmk_uname != NULL);
	pcmk_uname_len = strlen(pcmk_uname);

	if (pcmk_nodeid != 0) {
	/* Ensure the local node always exists */
	crm_get_peer(pcmk_nodeid, pcmk_uname);
	}

	if (our_uuid != NULL) {
	*our_uuid = get_corosync_uuid(pcmk_nodeid, pcmk_uname);
	}

	if (our_uname != NULL) {
	*our_uname = strdup(pcmk_uname);
	}

	if (nodeid != NULL) {
	*nodeid = pcmk_nodeid;
	}

	return TRUE;
	}

	gboolean
	check_message_sanity(const AIS_Message * msg, const char *data)
	{
	gboolean sane = TRUE;
	int dest = msg->host.type;
	int tmp_size = msg->header.size - sizeof(AIS_Message);

	if (sane && msg->header.size == 0) {
	crm_warn("Message with no size");
	sane = FALSE;
	}

	if (sane && msg->header.error != CS_OK) {
	crm_warn("Message header contains an error: %d", msg->header.error);
	sane = FALSE;
	}

	if (sane && ais_data_len(msg) != tmp_size) {
	crm_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg),
	tmp_size);
	sane = TRUE;
	}

	if (sane && ais_data_len(msg) == 0) {
	crm_warn("Message with no payload");
	sane = FALSE;
	}

	if (sane && data && msg->is_compressed == FALSE) {
	int str_size = strlen(data) + 1;

	if (ais_data_len(msg) != str_size) {
	int lpc = 0;

	crm_warn("Message payload is corrupted: expected %d bytes, got %d",
	ais_data_len(msg), str_size);
	sane = FALSE;
	for (lpc = (str_size - 10); lpc < msg->size; lpc++) {
	if (lpc < 0) {
	lpc = 0;
	}
	crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]);
	}
	}
	}

	if (sane == FALSE) {
	crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
	msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size);

	} else {
	crm_trace
	("Verfied message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
	msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)),
	msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed,
	ais_data_len(msg), msg->header.size);
	}

	return sane;
	}

	enum cluster_type_e
	find_corosync_variant(void)
	{
	int rc = CS_OK;
	cmap_handle_t handle;

	/* There can be only one (possibility if confdb isn't around) */
	rc = cmap_initialize(&handle);
	if (rc != CS_OK) {
	crm_info("Failed to initialize the cmap API. Error %d", rc);
	return pcmk_cluster_unknown;
	}

	cmap_finalize(handle);
	return pcmk_cluster_corosync;
	}

	gboolean
	crm_is_corosync_peer_active(const crm_node_t * node)
	{
	if (node == NULL) {
	crm_trace("NULL");
	return FALSE;

	} else if(safe_str_neq(node->state, CRM_NODE_MEMBER)) {
	crm_trace("%s: state=%s", node->uname, node->state);
	return FALSE;

	} else if((node->processes & crm_proc_cpg) == 0) {
	crm_trace("%s: processes=%.16x", node->uname, node->processes);
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	corosync_initialize_nodelist(void cluster, gboolean force_member, xmlNode xml_parent)
	{
	int lpc = 0;
	int rc = CS_OK;
	int retries = 0;
	gboolean any = FALSE;
	cmap_handle_t cmap_handle;

	do {
	rc = cmap_initialize(&cmap_handle);
	if(rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc), retries);
	sleep(retries);
	}

	} while(retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc);
	return FALSE;
	}

	crm_trace("Initializing corosync nodelist");
	for(lpc = 0; ; lpc++) {
	uint32_t nodeid = 0;
	char *name = NULL;
	char *key = NULL;

	key = g_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &nodeid);
	g_free(key);

	if(rc != CS_OK) {
	break;
	}

	name = corosync_node_name(cmap_handle, nodeid);

	if(nodeid > 0 \|\| name != NULL) {
	crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name);
	crm_get_peer(nodeid, name);
	}

	if(nodeid > 0 && name != NULL) {
	any = TRUE;

	if(xml_parent) {
	xmlNode *node = create_xml_node(xml_parent, XML_CIB_TAG_NODE);
	crm_xml_add_int(node, XML_ATTR_ID, nodeid);
	crm_xml_add(node, XML_ATTR_UNAME, name);
	if(force_member) {
	crm_xml_add(node, XML_ATTR_TYPE, CRM_NODE_MEMBER);
	}
	}
	}

	free(name);
	}
	cmap_finalize(cmap_handle);
	return any;
	}
	diff --git a/lib/cluster/legacy.c b/lib/cluster/legacy.c
	index 7efdd0839e..05ab0c601a 100644
	--- a/lib/cluster/legacy.c
	+++ b/lib/cluster/legacy.c
	@@ -1,1391 +1,1391 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/cluster/internal.h>
	#include <bzlib.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster.h>
	#include <crm/common/mainloop.h>
	#include <sys/utsname.h>

	#if SUPPORT_COROSYNC
	# include <corosync/confdb.h>
	# include <corosync/corodefs.h>
	# include <corosync/cpg.h>
	cpg_handle_t pcmk_cpg_handle = 0;

	struct cpg_name pcmk_cpg_group = {
	.length = 0,
	.value[0] = 0,
	};
	#endif

	#if HAVE_CMAP
	# include <corosync/cmap.h>
	#endif

	#if SUPPORT_CMAN
	# include <libcman.h>
	cman_handle_t pcmk_cman_handle = NULL;
	#endif

	static char *pcmk_uname = NULL;
	static int pcmk_uname_len = 0;
	static uint32_t pcmk_nodeid = 0;
	int ais_membership_timer = 0;
	gboolean ais_membership_force = FALSE;
	int ais_dispatch(gpointer user_data);

	#define cs_repeat(counter, max, code) do { \
	code; \
	if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	counter++; \
	crm_debug("Retrying operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while(counter < max)

	enum crm_ais_msg_types
	text2msg_type(const char *text)
	{
	int type = crm_msg_none;

	CRM_CHECK(text != NULL, return type);
	if (safe_str_eq(text, "ais")) {
	type = crm_msg_ais;
	} else if (safe_str_eq(text, "crm_plugin")) {
	type = crm_msg_ais;
	} else if (safe_str_eq(text, CRM_SYSTEM_CIB)) {
	type = crm_msg_cib;
	} else if (safe_str_eq(text, CRM_SYSTEM_CRMD)) {
	type = crm_msg_crmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_DC)) {
	type = crm_msg_crmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
	type = crm_msg_te;
	} else if (safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
	type = crm_msg_pe;
	} else if (safe_str_eq(text, CRM_SYSTEM_LRMD)) {
	type = crm_msg_lrmd;
	} else if (safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
	type = crm_msg_stonithd;
	} else if (safe_str_eq(text, "stonith-ng")) {
	type = crm_msg_stonith_ng;
	} else if (safe_str_eq(text, "attrd")) {
	type = crm_msg_attrd;

	} else {
	/* This will normally be a transient client rather than
	* a cluster daemon. Set the type to the pid of the client
	*/
	int scan_rc = sscanf(text, "%d", &type);

	if (scan_rc != 1) {
	/* Ensure its sane */
	type = crm_msg_none;
	}
	}
	return type;
	}

	char *
	get_ais_data(const AIS_Message * msg)
	{
	int rc = BZ_OK;
	char *uncompressed = NULL;
	unsigned int new_size = msg->size + 1;

	if (msg->is_compressed == FALSE) {
	crm_trace("Returning uncompressed message data");
	uncompressed = strdup(msg->data);

	} else {
	crm_trace("Decompressing message data");
	uncompressed = calloc(1, new_size);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, (char *)msg->data,
	msg->compressed_size, 1, 0);

	CRM_ASSERT(rc == BZ_OK);
	CRM_ASSERT(new_size == msg->size);
	}

	return uncompressed;
	}

	#if SUPPORT_COROSYNC
	int ais_fd_sync = -1;
	int ais_fd_async = -1; /* never send messages via this channel */
	void *ais_ipc_ctx = NULL;

	hdb_handle_t ais_ipc_handle = 0;
	static char *ais_cluster_name = NULL;

	gboolean
	get_ais_nodeid(uint32_t * id, char **uname)
	{
	struct iovec iov;
	int retries = 0;
	int rc = CS_OK;
	cs_ipc_header_response_t header;
	struct crm_ais_nodeid_resp_s answer;

	header.error = CS_OK;
	header.id = crm_class_nodeid;
	header.size = sizeof(cs_ipc_header_response_t);

	CRM_CHECK(id != NULL, return FALSE);
	CRM_CHECK(uname != NULL, return FALSE);

	iov.iov_base = &header;
	iov.iov_len = header.size;

	retry:
	errno = 0;
	rc = coroipcc_msg_send_reply_receive(ais_ipc_handle, &iov, 1, &answer, sizeof(answer));
	if (rc == CS_OK) {
	CRM_CHECK(answer.header.size == sizeof(struct crm_ais_nodeid_resp_s),
	crm_err("Odd message: id=%d, size=%d, error=%d",
	answer.header.id, answer.header.size, answer.header.error));
	CRM_CHECK(answer.header.id == crm_class_nodeid,
	crm_err("Bad response id: %d", answer.header.id));
	}

	if ((rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) && retries < 20) {
	retries++;
	crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries);
	sleep(retries); /* Proportional back off */
	goto retry;
	}

	if (rc != CS_OK) {
	crm_err("Sending nodeid request: FAILED (rc=%d): %s", rc, ais_error2text(rc));
	return FALSE;

	} else if (answer.header.error != CS_OK) {
	crm_err("Bad response from peer: (rc=%d): %s", rc, ais_error2text(rc));
	return FALSE;
	}

	crm_info("Server details: id=%u uname=%s cname=%s", answer.id, answer.uname, answer.cname);

	*id = answer.id;
	*uname = strdup(answer.uname);
	ais_cluster_name = strdup(answer.cname);

	return TRUE;
	}

	gboolean
	crm_get_cluster_name(char **cname)
	{
	CRM_CHECK(cname != NULL, return FALSE);
	if (ais_cluster_name) {
	*cname = strdup(ais_cluster_name);
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	send_ais_text(int class, const char *data,
	gboolean local, const char *node, enum crm_ais_msg_types dest)
	{
	static int msg_id = 0;
	static int local_pid = 0;
	enum cluster_type_e cluster_type = get_cluster_type();

	int retries = 0;
	int rc = CS_OK;
	int buf_len = sizeof(cs_ipc_header_response_t);

	char *buf = NULL;
	struct iovec iov;
	const char *transport = "pcmk";
	cs_ipc_header_response_t *header = NULL;
	AIS_Message *ais_msg = NULL;
	enum crm_ais_msg_types sender = text2msg_type(crm_system_name);

	/* There are only 6 handlers registered to crm_lib_service in plugin.c */
	CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class); return FALSE);

	if (data == NULL) {
	data = "";
	}

	if (local_pid == 0) {
	local_pid = getpid();
	}

	if (sender == crm_msg_none) {
	sender = local_pid;
	}

	ais_msg = calloc(1, sizeof(AIS_Message));

	ais_msg->id = msg_id++;
	ais_msg->header.id = class;
	ais_msg->header.error = CS_OK;

	ais_msg->host.type = dest;
	ais_msg->host.local = local;
	if (node) {
	ais_msg->host.size = strlen(node);
	memset(ais_msg->host.uname, 0, MAX_NAME);
	memcpy(ais_msg->host.uname, node, ais_msg->host.size);
	ais_msg->host.id = 0;

	} else {
	ais_msg->host.size = 0;
	memset(ais_msg->host.uname, 0, MAX_NAME);
	ais_msg->host.id = 0;
	}

	ais_msg->sender.id = 0;
	ais_msg->sender.type = sender;
	ais_msg->sender.pid = local_pid;
	ais_msg->sender.size = pcmk_uname_len;
	memset(ais_msg->sender.uname, 0, MAX_NAME);
	memcpy(ais_msg->sender.uname, pcmk_uname, ais_msg->sender.size);

	ais_msg->size = 1 + strlen(data);

	if (ais_msg->size < CRM_BZ2_THRESHOLD) {
	failback:
	ais_msg = realloc(ais_msg, sizeof(AIS_Message) + ais_msg->size);
	memcpy(ais_msg->data, data, ais_msg->size);

	} else {
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	unsigned int len = (ais_msg->size * 1.1) + 600; /* recomended size */

	crm_trace("Compressing message payload");
	compressed = malloc( len);

	rc = BZ2_bzBuffToBuffCompress(compressed, &len, uncompressed, ais_msg->size, CRM_BZ2_BLOCKS,
	0, CRM_BZ2_WORK);

	free(uncompressed);

	if (rc != BZ_OK) {
	crm_err("Compression failed: %d", rc);
	free(compressed);
	goto failback;
	}

	ais_msg = realloc(ais_msg, sizeof(AIS_Message) + len + 1);
	memcpy(ais_msg->data, compressed, len);
	ais_msg->data[len] = 0;
	free(compressed);

	ais_msg->is_compressed = TRUE;
	ais_msg->compressed_size = len;

	crm_trace("Compression details: %d -> %d", ais_msg->size, ais_data_len(ais_msg));
	}

	ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg);

	crm_trace("Sending%s message %d to %s.%s (data=%d, total=%d)",
	ais_msg->is_compressed ? " compressed" : "",
	ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(dest),
	ais_data_len(ais_msg), ais_msg->header.size);

	iov.iov_base = ais_msg;
	iov.iov_len = ais_msg->header.size;
	buf = realloc(buf, buf_len);

	do {
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	retries++;
	crm_info("Peer overloaded or membership in flux:"
	" Re-sending message (Attempt %d of 20)", retries);
	sleep(retries); /* Proportional back off */
	}

	errno = 0;
	switch (cluster_type) {
	case pcmk_cluster_corosync:
	CRM_ASSERT(FALSE/Not supported here/);
	break;

	case pcmk_cluster_classic_ais:
	rc = coroipcc_msg_send_reply_receive(ais_ipc_handle, &iov, 1, buf, buf_len);
	header = (cs_ipc_header_response_t *) buf;
	if (rc == CS_OK) {
	CRM_CHECK(header->size == sizeof(cs_ipc_header_response_t),
	crm_err("Odd message: id=%d, size=%d, class=%d, error=%d",
	header->id, header->size, class, header->error));

	CRM_ASSERT(buf_len >= header->size);
	CRM_CHECK(header->id == CRM_MESSAGE_IPC_ACK,
	crm_err("Bad response id (%d) for request (%d)", header->id,
	ais_msg->header.id));
	CRM_CHECK(header->error == CS_OK, rc = header->error);
	}
	break;

	case pcmk_cluster_cman:
	transport = "cpg";
	CRM_CHECK(dest != crm_msg_ais, rc = CS_ERR_MESSAGE_ERROR; goto bail);
	rc = cpg_mcast_joined(pcmk_cpg_handle, CPG_TYPE_AGREED, &iov, 1);
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED;
	int rc2 = cpg_flow_control_state_get(pcmk_cpg_handle, &fc_state);

	if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
	crm_warn("Connection overloaded, cannot send messages");
	goto bail;

	} else if (rc2 != CS_OK) {
	crm_warn("Could not determin the connection state: %s (%d)",
	ais_error2text(rc2), rc2);
	goto bail;
	}
	}
	break;

	case pcmk_cluster_unknown:
	case pcmk_cluster_invalid:
	case pcmk_cluster_heartbeat:
	CRM_ASSERT(is_openais_cluster());
	break;
	}

	} while ((rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) && retries < 20);

	bail:
	if (rc != CS_OK) {
	crm_perror(LOG_ERR, "Sending message %d via %s: FAILED (rc=%d): %s",
	ais_msg->id, transport, rc, ais_error2text(rc));

	} else {
	crm_trace("Message %d: sent", ais_msg->id);
	}

	free(buf);
	free(ais_msg);
	return (rc == CS_OK);
	}

	gboolean
	send_ais_message(xmlNode * msg, gboolean local, const char *node, enum crm_ais_msg_types dest)
	{
	gboolean rc = TRUE;
	char *data = NULL;

	if (is_classic_ais_cluster()) {
	if (ais_fd_async < 0) {
	crm_err("Not connected to AIS: %d", ais_fd_async);
	return FALSE;
	}
	}

	data = dump_xml_unformatted(msg);
	rc = send_ais_text(crm_class_cluster, data, local, node, dest);
	free(data);
	return rc;
	}

	void
	terminate_ais_connection(void)
	{
	crm_notice("Disconnecting from Corosync");

	if (is_classic_ais_cluster()) {
	if(ais_ipc_handle) {
	crm_trace("Disconnecting plugin");
	coroipcc_service_disconnect(ais_ipc_handle);
	ais_ipc_handle = 0;
	} else {
	crm_info("No plugin connection");
	}

	} else {
	if(pcmk_cpg_handle) {
	crm_trace("Disconnecting CPG");
	cpg_leave(pcmk_cpg_handle, &pcmk_cpg_group);
	cpg_finalize(pcmk_cpg_handle);
	pcmk_cpg_handle = 0;

	} else {
	crm_info("No CPG connection");
	}
	}

	# if SUPPORT_CMAN
	if (is_cman_cluster()) {
	if(pcmk_cman_handle) {
	crm_trace("Disconnecting cman");
	cman_stop_notification(pcmk_cman_handle);
	cman_finish(pcmk_cman_handle);

	} else {
	crm_info("No cman connection");
	}
	}
	# endif
	ais_fd_async = -1;
	ais_fd_sync = -1;
	}


	static crm_node_t *
	crm_update_ais_node(xmlNode * member, long long seq)
	{
	const char *id_s = crm_element_value(member, "id");
	const char *addr = crm_element_value(member, "addr");
	const char *uname = crm_element_value(member, "uname");
	const char *state = crm_element_value(member, "state");
	const char *born_s = crm_element_value(member, "born");
	const char *seen_s = crm_element_value(member, "seen");
	const char *votes_s = crm_element_value(member, "votes");
	const char *procs_s = crm_element_value(member, "processes");

	int votes = crm_int_helper(votes_s, NULL);
	unsigned int id = crm_int_helper(id_s, NULL);
	unsigned int procs = crm_int_helper(procs_s, NULL);

	/* TODO: These values will contain garbage if version < 0.7.1 */
	uint64_t born = crm_int_helper(born_s, NULL);
	uint64_t seen = crm_int_helper(seen_s, NULL);

	return crm_update_peer(__FUNCTION__, id, born, seen, votes, procs, uname, uname, addr, state);
	}

	static gboolean
	ais_dispatch_message(AIS_Message * msg, gboolean(dispatch) (AIS_Message , char *, int))
	{
	char *data = NULL;
	char *uncompressed = NULL;

	xmlNode *xml = NULL;

	CRM_ASSERT(msg != NULL);

	crm_trace("Got new%s message (size=%d, %d, %d)",
	msg->is_compressed ? " compressed" : "",
	ais_data_len(msg), msg->size, msg->compressed_size);

	data = msg->data;
	if (msg->is_compressed && msg->size > 0) {
	int rc = BZ_OK;
	unsigned int new_size = msg->size + 1;

	if (check_message_sanity(msg, NULL) == FALSE) {
	goto badmsg;
	}

	crm_trace("Decompressing message data");
	uncompressed = calloc(1, new_size);
	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, data, msg->compressed_size, 1, 0);

	if (rc != BZ_OK) {
	crm_err("Decompression failed: %d", rc);
	goto badmsg;
	}

	CRM_ASSERT(rc == BZ_OK);
	CRM_ASSERT(new_size == msg->size);

	data = uncompressed;

	} else if (check_message_sanity(msg, data) == FALSE) {
	goto badmsg;

	} else if (safe_str_eq("identify", data)) {
	int pid = getpid();
	char *pid_s = crm_itoa(pid);

	send_ais_text(crm_class_cluster, pid_s, TRUE, NULL, crm_msg_ais);
	free(pid_s);
	goto done;
	}

	if (msg->header.id != crm_class_members) {
	crm_get_peer(msg->sender.id, msg->sender.uname);
	}

	if (msg->header.id == crm_class_rmpeer) {
	uint32_t id = crm_int_helper(data, NULL);

	crm_info("Removing peer %s/%u", data, id);
	reap_crm_member(id);
	goto done;

	} else if (is_classic_ais_cluster()) {
	if (msg->header.id == crm_class_members \|\| msg->header.id == crm_class_quorum) {
	xmlNode *node = NULL;
	const char *value = NULL;
	gboolean quorate = FALSE;

	xml = string2xml(data);
	if (xml == NULL) {
	crm_err("Invalid membership update: %s", data);
	goto badmsg;
	}

	value = crm_element_value(xml, "quorate");
	CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No quorum value:"); goto badmsg);
	if (crm_is_true(value)) {
	quorate = TRUE;
	}

	value = crm_element_value(xml, "id");
	CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No membership id"); goto badmsg);
	crm_peer_seq = crm_int_helper(value, NULL);

	if (quorate != crm_have_quorum) {
	crm_notice("Membership %s: quorum %s", value, quorate ? "acquired" : "lost");
	crm_have_quorum = quorate;

	} else {
	crm_info("Membership %s: quorum %s", value, quorate ? "retained" : "still lost");
	}

	for (node = __xml_first_child(xml); node != NULL; node = __xml_next(node)) {
	crm_update_ais_node(node, crm_peer_seq);
	}
	}
	}

	crm_trace("Payload: %s", data);
	if (dispatch != NULL) {
	dispatch(msg, data, 0);
	}

	done:
	free(uncompressed);
	free_xml(xml);
	return TRUE;

	badmsg:
	crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
	" min=%d, total=%d, size=%d, bz2_size=%d",
	msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	msg->sender.pid, (int)sizeof(AIS_Message),
	msg->header.size, msg->size, msg->compressed_size);
	goto done;
	}

	int
	ais_dispatch(gpointer user_data)
	{
	int rc = CS_OK;
	gboolean good = TRUE;

	gboolean(dispatch) (AIS_Message , char *, int) = user_data;

	do {
	char *buffer = NULL;

	rc = coroipcc_dispatch_get(ais_ipc_handle, (void **)&buffer, 0);
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	return 0;
	}
	if (rc != CS_OK) {
	crm_perror(LOG_ERR, "Receiving message body failed: (%d) %s", rc, ais_error2text(rc));
	return -1;
	}
	if (buffer == NULL) {
	/* NULL is a legal "no message afterall" value */
	return 0;
	}
	good = ais_dispatch_message((AIS_Message *) buffer, dispatch);
	coroipcc_dispatch_put(ais_ipc_handle);

	} while (good && ais_ipc_handle);

	if(good) {
	return 0;
	}

	return -1;
	}

	static void
	ais_destroy(gpointer user_data)
	{
	crm_err("AIS connection terminated");
	ais_fd_sync = -1;
	exit(1);
	}


	# if SUPPORT_CMAN

	static int
	pcmk_cman_dispatch(gpointer user_data)
	{
	int rc = cman_dispatch(pcmk_cman_handle, CMAN_DISPATCH_ALL);

	if (rc < 0) {
	crm_err("Connection to cman failed: %d", rc);
	return FALSE;
	}
	return TRUE;
	}

	# define MAX_NODES 256

	static void
	cman_event_callback(cman_handle_t handle, void *privdata, int reason, int arg)
	{
	int rc = 0, lpc = 0, node_count = 0;

	cman_cluster_t cluster;
	static cman_node_t cman_nodes[MAX_NODES];

	gboolean(*dispatch) (unsigned long long, gboolean) = privdata;

	switch (reason) {
	case CMAN_REASON_STATECHANGE:

	memset(&cluster, 0, sizeof(cluster));
	rc = cman_get_cluster(pcmk_cman_handle, &cluster);
	if (rc < 0) {
	crm_err("Couldn't query cman cluster details: %d %d", rc, errno);
	return;
	}

	crm_peer_seq = cluster.ci_generation;
	if (arg != crm_have_quorum) {
	crm_notice("Membership %llu: quorum %s", crm_peer_seq, arg ? "acquired" : "lost");
	crm_have_quorum = arg;

	} else {
	crm_info("Membership %llu: quorum %s", crm_peer_seq,
	arg ? "retained" : "still lost");
	}

	rc = cman_get_nodes(pcmk_cman_handle, MAX_NODES, &node_count, cman_nodes);
	if (rc < 0) {
	crm_err("Couldn't query cman node list: %d %d", rc, errno);
	return;
	}

	for (lpc = 0; lpc < node_count; lpc++) {
	if (cman_nodes[lpc].cn_nodeid == 0) {
	/* Never allow node ID 0 to be considered a member #315711 */
	cman_nodes[lpc].cn_member = 0;
	}
	crm_update_peer(__FUNCTION__, cman_nodes[lpc].cn_nodeid, cman_nodes[lpc].cn_incarnation,
	cman_nodes[lpc].cn_member ? crm_peer_seq : 0, 0, 0,
	cman_nodes[lpc].cn_name, cman_nodes[lpc].cn_name, NULL,
	cman_nodes[lpc].cn_member ? CRM_NODE_MEMBER : CRM_NODE_LOST);
	}

	if (dispatch) {
	dispatch(crm_peer_seq, crm_have_quorum);
	}
	break;

	case CMAN_REASON_TRY_SHUTDOWN:
	/* Always reply with a negative - pacemaker needs to be stopped first */
	crm_info("CMAN wants to shut down: %s", arg ? "forced" : "optional");
	cman_replyto_shutdown(pcmk_cman_handle, 0);
	break;

	case CMAN_REASON_CONFIG_UPDATE:
	/* Ignore */
	break;
	}
	}
	# endif

	gboolean
	init_cman_connection(gboolean(dispatch) (unsigned long long, gboolean), void (destroy) (gpointer))
	{
	# if SUPPORT_CMAN
	int rc = -1, fd = -1;
	cman_cluster_t cluster;
	struct mainloop_fd_callbacks cman_fd_callbacks = {
	.dispatch = pcmk_cman_dispatch,
	.destroy = destroy,
	};

	crm_info("Configuring Pacemaker to obtain quorum from cman");

	memset(&cluster, 0, sizeof(cluster));

	pcmk_cman_handle = cman_init(dispatch);
	if (pcmk_cman_handle == NULL \|\| cman_is_active(pcmk_cman_handle) == FALSE) {
	crm_err("Couldn't connect to cman");
	goto cman_bail;
	}

	rc = cman_get_cluster(pcmk_cman_handle, &cluster);
	if (rc < 0) {
	crm_err("Couldn't query cman cluster details: %d %d", rc, errno);
	goto cman_bail;
	}
	ais_cluster_name = strdup(cluster.ci_name);

	rc = cman_start_notification(pcmk_cman_handle, cman_event_callback);
	if (rc < 0) {
	crm_err("Couldn't register for cman notifications: %d %d", rc, errno);
	goto cman_bail;
	}

	/* Get the current membership state */
	cman_event_callback(pcmk_cman_handle, dispatch, CMAN_REASON_STATECHANGE,
	cman_is_quorate(pcmk_cman_handle));

	fd = cman_get_fd(pcmk_cman_handle);

	- mainloop_add_fd("cman", fd, dispatch, &cman_fd_callbacks);
	+ mainloop_add_fd("cman", G_PRIORITY_MEDIUM, fd, dispatch, &cman_fd_callbacks);

	cman_bail:
	if (rc < 0) {
	cman_finish(pcmk_cman_handle);
	return FALSE;
	}
	# else
	crm_err("cman qorum is not supported in this build");
	exit(100);
	# endif
	return TRUE;
	}

	# ifdef SUPPORT_COROSYNC
	gboolean(pcmk_cpg_dispatch_fn) (AIS_Message , char *, int) = NULL;

	static int
	pcmk_cpg_dispatch(gpointer user_data)
	{
	int rc = 0;

	pcmk_cpg_dispatch_fn = user_data;
	rc = cpg_dispatch(pcmk_cpg_handle, CS_DISPATCH_ALL);
	if (rc != CS_OK) {
	crm_err("Connection to the CPG API failed: %d", rc);
	return -1;
	}
	return 0;
	}

	static void
	pcmk_cpg_deliver(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	AIS_Message ais_msg = (AIS_Message ) msg;

	if (ais_msg->sender.id > 0 && ais_msg->sender.id != nodeid) {
	crm_err("Nodeid mismatch from %d.%d: claimed nodeid=%u", nodeid, pid, ais_msg->sender.id);
	return;

	} else if (ais_msg->host.size != 0 && safe_str_neq(ais_msg->host.uname, pcmk_uname)) {
	/* Not for us */
	return;
	}

	ais_msg->sender.id = nodeid;
	if (ais_msg->sender.size == 0) {
	crm_node_t *peer = crm_get_peer(nodeid, NULL);

	if (peer == NULL) {
	crm_err("Peer with nodeid=%u is unknown", nodeid);

	} else if (peer->uname == NULL) {
	crm_err("No uname for peer with nodeid=%u", nodeid);

	} else {
	crm_notice("Fixing uname for peer with nodeid=%u", nodeid);
	ais_msg->sender.size = strlen(peer->uname);
	memset(ais_msg->sender.uname, 0, MAX_NAME);
	memcpy(ais_msg->sender.uname, peer->uname, ais_msg->sender.size);
	}
	}

	ais_dispatch_message(ais_msg, pcmk_cpg_dispatch_fn);
	}

	static void
	pcmk_cpg_membership(cpg_handle_t handle,
	const struct cpg_name *groupName,
	const struct cpg_address *member_list, size_t member_list_entries,
	const struct cpg_address *left_list, size_t left_list_entries,
	const struct cpg_address *joined_list, size_t joined_list_entries)
	{
	int i;

	for (i = 0; i < member_list_entries; i++) {
	crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL);
	crm_debug("Member[%d] %d ", i, member_list[i].nodeid);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
	}

	for (i = 0; i < left_list_entries; i++) {
	crm_node_t *peer = crm_get_peer(left_list[i].nodeid, NULL);
	crm_debug("Left[%d] %d ", i, left_list[i].nodeid);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, OFFLINESTATUS);
	}
	}

	cpg_callbacks_t cpg_callbacks = {
	.cpg_deliver_fn = pcmk_cpg_deliver,
	.cpg_confchg_fn = pcmk_cpg_membership,
	};
	# endif

	static gboolean
	init_cpg_connection(gboolean(dispatch) (AIS_Message , char , int), void (destroy) (gpointer),
	uint32_t * nodeid)
	{
	# ifdef SUPPORT_COROSYNC
	int rc = -1;
	int fd = 0;
	int retries = 0;
	crm_node_t *peer = NULL;
	struct mainloop_fd_callbacks cpg_fd_callbacks = {
	.dispatch = pcmk_cpg_dispatch,
	.destroy = destroy,
	};

	strcpy(pcmk_cpg_group.value, crm_system_name);
	pcmk_cpg_group.length = strlen(crm_system_name) + 1;

	cs_repeat(retries, 30, rc = cpg_initialize(&pcmk_cpg_handle, &cpg_callbacks));
	if (rc != CS_OK) {
	crm_err("Could not connect to the Cluster Process Group API: %d\n", rc);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_local_get(pcmk_cpg_handle, (unsigned int *)nodeid));
	if (rc != CS_OK) {
	crm_err("Could not get local node id from the CPG API");
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_join(pcmk_cpg_handle, &pcmk_cpg_group));
	if (rc != CS_OK) {
	crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
	goto bail;
	}

	rc = cpg_fd_get(pcmk_cpg_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %d\n", rc);
	goto bail;
	}

	- mainloop_add_fd("corosync-cpg", fd, dispatch, &cpg_fd_callbacks);
	+ mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, dispatch, &cpg_fd_callbacks);

	bail:
	if (rc != CS_OK) {
	cpg_finalize(pcmk_cpg_handle);
	return FALSE;
	}

	peer = crm_get_peer(pcmk_nodeid, pcmk_uname);
	crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);

	# else
	crm_err("The Corosync CPG API is not supported in this build");
	exit(100);
	# endif
	return TRUE;
	}

	gboolean
	init_quorum_connection(gboolean(*dispatch) (unsigned long long, gboolean),
	void (*destroy) (gpointer))
	{
	crm_err("The Corosync quorum API is not supported in this build");
	exit(100);
	return TRUE;
	}

	static gboolean
	init_ais_connection_classic(gboolean(dispatch) (AIS_Message , char *, int),
	void (destroy) (gpointer), char our_uuid, char *our_uname,
	int *nodeid)
	{
	int rc;
	int pid = 0;
	char *pid_s = NULL;
	struct utsname name;
	struct mainloop_fd_callbacks ais_fd_callbacks = {
	.dispatch = ais_dispatch,
	.destroy = destroy,
	};

	crm_info("Creating connection to our Corosync plugin");
	rc = coroipcc_service_connect(COROSYNC_SOCKET_NAME, PCMK_SERVICE_ID,
	AIS_IPC_MESSAGE_SIZE, AIS_IPC_MESSAGE_SIZE, AIS_IPC_MESSAGE_SIZE,
	&ais_ipc_handle);
	if (ais_ipc_handle) {
	coroipcc_fd_get(ais_ipc_handle, &ais_fd_async);
	} else {
	crm_info("Connection to our AIS plugin (%d) failed: %s (%d)",
	PCMK_SERVICE_ID, strerror(errno), errno);
	return FALSE;
	}
	if (ais_fd_async <= 0 && rc == CS_OK) {
	crm_err("No context created, but connection reported 'ok'");
	rc = CS_ERR_LIBRARY;
	}
	if (rc != CS_OK) {
	crm_info("Connection to our AIS plugin (%d) failed: %s (%d)", PCMK_SERVICE_ID,
	ais_error2text(rc), rc);
	}

	if (rc != CS_OK) {
	return FALSE;
	}

	if (destroy == NULL) {
	destroy = ais_destroy;
	}

	- mainloop_add_fd("corosync-plugin", ais_fd_async, dispatch, &ais_fd_callbacks);
	+ mainloop_add_fd("corosync-plugin", G_PRIORITY_MEDIUM, ais_fd_async, dispatch, &ais_fd_callbacks);
	crm_info("AIS connection established");

	pid = getpid();
	pid_s = crm_itoa(pid);
	send_ais_text(crm_class_cluster, pid_s, TRUE, NULL, crm_msg_ais);
	free(pid_s);

	if (uname(&name) < 0) {
	crm_perror(LOG_ERR, "Could not determin the current host");
	exit(100);
	}

	get_ais_nodeid(&pcmk_nodeid, &pcmk_uname);
	if (safe_str_neq(name.nodename, pcmk_uname)) {
	crm_crit("Node name mismatch! OpenAIS supplied %s, our lookup returned %s",
	pcmk_uname, name.nodename);
	crm_notice
	("Node name mismatches usually occur when assigned automatically by DHCP servers");
	crm_notice("If this node was part of the cluster with a different name,"
	" you will need to remove the old entry with crm_node --remove");
	}
	return TRUE;
	}


	static int
	pcmk_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	if (msg && is_classic_ais_cluster()) {
	xmlNode *node = NULL;

	for (node = __xml_first_child(msg); node != NULL; node = __xml_next(node)) {
	int id = 0;
	int children = 0;
	const char *uname = crm_element_value(node, "uname");

	crm_element_value_int(node, "id", &id);
	crm_element_value_int(node, "processes", &children);
	if (id == 0) {
	crm_log_xml_err(msg, "Bad Update");
	} else {
	crm_node_t *peer = crm_get_peer(id, uname);
	crm_update_peer_proc(__FUNCTION__, peer, children, NULL);
	}
	}
	}

	free_xml(msg);
	return 0;
	}

	static void
	pcmk_mcp_destroy(gpointer user_data)
	{
	void (*callback)(gpointer data) = user_data;
	if(callback) {
	callback(NULL);
	}
	}

	gboolean
	init_ais_connection(gboolean(dispatch) (AIS_Message , char , int), void (destroy) (gpointer),
	char our_uuid, char our_uname, int *nodeid)
	{
	int retries = 0;
	static struct ipc_client_callbacks mcp_callbacks =
	{
	.dispatch = pcmk_mcp_dispatch,
	.destroy = pcmk_mcp_destroy
	};

	while (retries++ < 30) {
	int rc = init_ais_connection_once(dispatch, destroy, our_uuid, our_uname, nodeid);

	switch (rc) {
	case CS_OK:
	if (getenv("HA_mcp")) {
	xmlNode *poke = create_xml_node(NULL, "poke");
	mainloop_io_t *ipc = mainloop_add_ipc_client(CRM_SYSTEM_MCP, 0, destroy, &mcp_callbacks);
	crm_ipc_send(mainloop_get_ipc_client(ipc), poke, NULL, 0);
	free_xml(poke);
	}
	return TRUE;
	break;
	case CS_ERR_TRY_AGAIN:
	case CS_ERR_QUEUE_FULL:
	break;
	default:
	return FALSE;
	}
	}

	crm_err("Retry count exceeded: %d", retries);
	return FALSE;
	}

	static char *
	get_local_node_name(void)
	{
	char *name = NULL;
	struct utsname res;

	if (is_cman_cluster()) {
	# if SUPPORT_CMAN
	cman_node_t us;
	cman_handle_t cman;

	cman = cman_init(NULL);
	if (cman != NULL && cman_is_active(cman)) {
	us.cn_name[0] = 0;
	cman_get_node(cman, CMAN_NODEID_US, &us);
	name = strdup(us.cn_name);
	crm_info("Using CMAN node name: %s", name);

	} else {
	crm_err("Couldn't determin node name from CMAN");
	}

	cman_finish(cman);
	# endif

	} else if (uname(&res) < 0) {
	crm_perror(LOG_ERR, "Could not determin the current host");
	exit(100);

	} else {
	name = strdup(res.nodename);
	}
	return name;
	}

	extern int set_cluster_type(enum cluster_type_e type);

	gboolean
	init_ais_connection_once(gboolean(dispatch) (AIS_Message , char *, int),
	void (destroy) (gpointer), char our_uuid, char our_uname, int nodeid)
	{
	enum cluster_type_e stack = get_cluster_type();

	crm_peer_init();

	/* Here we just initialize comms */
	switch (stack) {
	case pcmk_cluster_classic_ais:
	if (init_ais_connection_classic(dispatch, destroy, our_uuid, &pcmk_uname, nodeid) ==
	FALSE) {
	return FALSE;
	}
	break;
	case pcmk_cluster_cman:
	if (init_cpg_connection(dispatch, destroy, &pcmk_nodeid) == FALSE) {
	return FALSE;
	}
	pcmk_uname = get_local_node_name();
	break;
	case pcmk_cluster_heartbeat:
	crm_info("Could not find an active corosync based cluster");
	return FALSE;
	break;
	default:
	crm_err("Invalid cluster type: %s (%d)", name_for_cluster_type(stack), stack);
	return FALSE;
	break;
	}

	crm_info("Connection to '%s': established", name_for_cluster_type(stack));

	CRM_ASSERT(pcmk_uname != NULL);
	pcmk_uname_len = strlen(pcmk_uname);

	if (pcmk_nodeid != 0) {
	/* Ensure the local node always exists */
	crm_get_peer(pcmk_nodeid, pcmk_uname);
	}

	if (our_uuid != NULL) {
	*our_uuid = get_corosync_uuid(pcmk_nodeid, pcmk_uname);
	}

	if (our_uname != NULL) {
	*our_uname = strdup(pcmk_uname);
	}

	if (nodeid != NULL) {
	*nodeid = pcmk_nodeid;
	}

	return TRUE;
	}

	gboolean
	check_message_sanity(const AIS_Message * msg, const char *data)
	{
	gboolean sane = TRUE;
	gboolean repaired = FALSE;
	int dest = msg->host.type;
	int tmp_size = msg->header.size - sizeof(AIS_Message);

	if (sane && msg->header.size == 0) {
	crm_warn("Message with no size");
	sane = FALSE;
	}

	if (sane && msg->header.error != CS_OK) {
	crm_warn("Message header contains an error: %d", msg->header.error);
	sane = FALSE;
	}

	if (sane && ais_data_len(msg) != tmp_size) {
	crm_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg),
	tmp_size);
	sane = TRUE;
	}

	if (sane && ais_data_len(msg) == 0) {
	crm_warn("Message with no payload");
	sane = FALSE;
	}

	if (sane && data && msg->is_compressed == FALSE) {
	int str_size = strlen(data) + 1;

	if (ais_data_len(msg) != str_size) {
	int lpc = 0;

	crm_warn("Message payload is corrupted: expected %d bytes, got %d",
	ais_data_len(msg), str_size);
	sane = FALSE;
	for (lpc = (str_size - 10); lpc < msg->size; lpc++) {
	if (lpc < 0) {
	lpc = 0;
	}
	crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]);
	}
	}
	}

	if (sane == FALSE) {
	crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
	msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size);

	} else if (repaired) {
	crm_err
	("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
	msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)),
	msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed,
	ais_data_len(msg), msg->header.size);
	} else {
	crm_trace
	("Verfied message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
	msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)),
	msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed,
	ais_data_len(msg), msg->header.size);
	}

	return sane;
	}
	#endif

	static int
	get_config_opt(confdb_handle_t config,
	hdb_handle_t object_handle, const char key, char value, const char fallback)
	{
	size_t len = 0;
	char *env_key = NULL;
	const char *env_value = NULL;
	char buffer[256];

	if (*value) {
	free(*value);
	*value = NULL;
	}

	if (object_handle > 0) {
	if (CS_OK == confdb_key_get(config, object_handle, key, strlen(key), &buffer, &len)) {
	*value = strdup(buffer);
	}
	}

	if (*value) {
	crm_info("Found '%s' for option: %s", *value, key);
	return 0;
	}

	env_key = crm_concat("HA", key, '_');
	env_value = getenv(env_key);
	free(env_key);

	if (*value) {
	crm_info("Found '%s' in ENV for option: %s", *value, key);
	*value = strdup(env_value);
	return 0;
	}

	if (fallback) {
	crm_info("Defaulting to '%s' for option: %s", fallback, key);
	*value = strdup(fallback);

	} else {
	crm_info("No default for option: %s", key);
	}

	return -1;
	}

	static confdb_handle_t
	config_find_init(confdb_handle_t config)
	{
	cs_error_t rc = CS_OK;
	confdb_handle_t local_handle = OBJECT_PARENT_HANDLE;

	rc = confdb_object_find_start(config, local_handle);
	if (rc == CS_OK) {
	return local_handle;
	} else {
	crm_err("Couldn't create search context: %d", rc);
	}
	return 0;
	}

	static hdb_handle_t
	config_find_next(confdb_handle_t config, const char *name, confdb_handle_t top_handle)
	{
	cs_error_t rc = CS_OK;
	hdb_handle_t local_handle = 0;

	if (top_handle == 0) {
	crm_err("Couldn't search for %s: no valid context", name);
	return 0;
	}

	crm_trace("Searching for %s in " HDB_X_FORMAT, name, top_handle);
	rc = confdb_object_find(config, top_handle, name, strlen(name), &local_handle);
	if (rc != CS_OK) {
	crm_info("No additional configuration supplied for: %s", name);
	local_handle = 0;
	} else {
	crm_info("Processing additional %s options...", name);
	}
	return local_handle;
	}

	enum cluster_type_e
	find_corosync_variant(void)
	{
	confdb_handle_t config;
	enum cluster_type_e found = pcmk_cluster_unknown;

	int rc;
	char *value = NULL;
	confdb_handle_t top_handle = 0;
	hdb_handle_t local_handle = 0;
	static confdb_callbacks_t callbacks = { };

	rc = confdb_initialize(&config, &callbacks);
	if (rc != CS_OK) {
	crm_debug("Could not initialize Cluster Configuration Database API instance error %d", rc);
	return found;
	}

	top_handle = config_find_init(config);
	local_handle = config_find_next(config, "service", top_handle);
	while (local_handle) {
	get_config_opt(config, local_handle, "name", &value, NULL);
	if (safe_str_eq("pacemaker", value)) {
	found = pcmk_cluster_classic_ais;

	get_config_opt(config, local_handle, "ver", &value, "0");
	crm_trace("Found Pacemaker plugin version: %s", value);
	break;
	}

	local_handle = config_find_next(config, "service", top_handle);
	}

	if (found == pcmk_cluster_unknown) {
	top_handle = config_find_init(config);
	local_handle = config_find_next(config, "quorum", top_handle);
	get_config_opt(config, local_handle, "provider", &value, NULL);

	if (safe_str_eq("quorum_cman", value)) {
	crm_trace("Found CMAN quorum provider");
	found = pcmk_cluster_cman;
	}
	}
	free(value);

	confdb_finalize(config);
	return found;
	}

	gboolean
	crm_is_corosync_peer_active(const crm_node_t * node)
	{
	enum crm_proc_flag proc = crm_proc_none;
	if (node == NULL) {
	crm_trace("NULL");
	return FALSE;

	} else if(safe_str_neq(node->state, CRM_NODE_MEMBER)) {
	crm_trace("%s: state=%s", node->uname, node->state);
	return FALSE;

	} else if(is_cman_cluster() && (node->processes & crm_proc_cpg)) {
	/* If we can still talk to our peer process on that node,
	* then its also part of the corosync membership
	*/
	crm_trace("%s: processes=%.16x", node->uname, node->processes);
	return TRUE;

	} else if(is_classic_ais_cluster() && (node->processes & crm_proc_plugin) == 0) {
	crm_trace("%s: processes=%.16x", node->uname, node->processes);
	return FALSE;
	}

	proc = text2proc(crm_system_name);
	if(proc != crm_proc_none && (node->processes & proc) == 0) {
	crm_trace("%s: proc %.16x not in %.16x", node->uname, proc, node->processes);
	return FALSE;
	}

	return TRUE;
	}
	diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
	index 9063a2797c..fd0b8154f0 100644
	--- a/lib/common/mainloop.c
	+++ b/lib/common/mainloop.c
	@@ -1,773 +1,773 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <stdlib.h>
	#include <signal.h>
	#include <errno.h>

	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>

	struct mainloop_child_s {
	pid_t pid;
	char *desc;
	unsigned timerid;
	unsigned watchid;
	gboolean timeout;
	void *privatedata;

	/* Called when a process dies */
	void (callback)(mainloop_child_t p, int status, int signo, int exitcode);
	};

	struct trigger_s {
	GSource source;
	gboolean running;
	gboolean trigger;
	void *user_data;
	guint id;

	};

	static gboolean
	crm_trigger_prepare(GSource * source, gint * timeout)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	/* cluster-glue's FD and IPC related sources make use of
	* g_source_add_poll() but do not set a timeout in their prepare
	* functions
	*
	* This means mainloop's poll() will block until an event for one
	* of these sources occurs - any /other/ type of source, such as
	* this one or g_idle_*, that doesn't use g_source_add_poll() is
	* S-O-L and wont be processed until there is something fd-based
	* happens.
	*
	* Luckily the timeout we can set here affects all sources and
	* puts an upper limit on how long poll() can take.
	*
	* So unconditionally set a small-ish timeout, not too small that
	* we're in constant motion, which will act as an upper bound on
	* how long the signal handling might be delayed for.
	*/
	timeout = 500; / Timeout in ms */

	return trig->trigger;
	}

	static gboolean
	crm_trigger_check(GSource * source)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	return trig->trigger;
	}

	static gboolean
	crm_trigger_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	int rc = TRUE;
	crm_trigger_t trig = (crm_trigger_t ) source;

	if(trig->running) {
	/* Wait until the existing job is complete before starting the next one */
	return TRUE;
	}
	trig->trigger = FALSE;

	if (callback) {
	rc = callback(trig->user_data);
	if(rc < 0) {
	crm_trace("Trigger handler %p not yet complete", trig);
	trig->running = TRUE;
	rc = TRUE;
	}
	}
	return rc;
	}

	static GSourceFuncs crm_trigger_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_trigger_dispatch,
	NULL
	};

	static crm_trigger_t *
	mainloop_setup_trigger(GSource * source, int priority, int(*dispatch) (gpointer user_data),
	gpointer userdata)
	{
	crm_trigger_t *trigger = NULL;

	trigger = (crm_trigger_t *) source;

	trigger->id = 0;
	trigger->trigger = FALSE;
	trigger->user_data = userdata;

	if (dispatch) {
	g_source_set_callback(source, dispatch, trigger, NULL);
	}

	g_source_set_priority(source, priority);
	g_source_set_can_recurse(source, FALSE);

	trigger->id = g_source_attach(source, NULL);
	return trigger;
	}

	void
	mainloop_trigger_complete(crm_trigger_t *trig)
	{
	crm_trace("Trigger handler %p complete", trig);
	trig->running = FALSE;
	}

	/* If dispatch returns:
	* -1: Job running but not complete
	* 0: Remove the trigger from mainloop
	* 1: Leave the trigger in mainloop
	*/
	crm_trigger_t *
	mainloop_add_trigger(int priority, int(*dispatch) (gpointer user_data), gpointer userdata)
	{
	GSource *source = NULL;

	CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
	source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
	CRM_ASSERT(source != NULL);

	return mainloop_setup_trigger(source, priority, dispatch, userdata);
	}

	void
	mainloop_set_trigger(crm_trigger_t * source)
	{
	source->trigger = TRUE;
	}

	gboolean
	mainloop_destroy_trigger(crm_trigger_t * source)
	{
	source->trigger = FALSE;
	if (source->id > 0) {
	g_source_remove(source->id);
	}
	return TRUE;
	}

	typedef struct signal_s {
	crm_trigger_t trigger; /* must be first */
	void (*handler) (int sig);
	int signal;

	} crm_signal_t;

	static crm_signal_t *crm_signals[NSIG];

	static gboolean
	crm_signal_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	crm_signal_t sig = (crm_signal_t ) source;

	crm_info("Invoking handler for signal %d: %s", sig->signal, strsignal(sig->signal));

	sig->trigger.trigger = FALSE;
	if (sig->handler) {
	sig->handler(sig->signal);
	}
	return TRUE;
	}

	static void
	mainloop_signal_handler(int sig)
	{
	if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
	mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
	}
	}

	static GSourceFuncs crm_signal_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_signal_dispatch,
	NULL
	};

	gboolean
	crm_signal(int sig, void (*dispatch) (int sig))
	{
	sigset_t mask;
	struct sigaction sa;
	struct sigaction old;

	if (sigemptyset(&mask) < 0) {
	crm_perror(LOG_ERR, "Call to sigemptyset failed");
	return FALSE;
	}

	memset(&sa, 0, sizeof(struct sigaction));
	sa.sa_handler = dispatch;
	sa.sa_flags = SA_RESTART;
	sa.sa_mask = mask;

	if (sigaction(sig, &sa, &old) < 0) {
	crm_perror(LOG_ERR, "Could not install signal handler for signal %d", sig);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	mainloop_add_signal(int sig, void (*dispatch) (int sig))
	{
	GSource *source = NULL;
	int priority = G_PRIORITY_HIGH - 1;

	if (sig == SIGTERM) {
	/* TERM is higher priority than other signals,
	* signals are higher priority than other ipc.
	* Yes, minus: smaller is "higher"
	*/
	priority--;
	}

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signals[sig] != NULL) {
	crm_err("Signal handler for %d is already installed", sig);
	return FALSE;
	}

	CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
	source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));

	crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
	CRM_ASSERT(crm_signals[sig] != NULL);

	crm_signals[sig]->handler = dispatch;
	crm_signals[sig]->signal = sig;

	if (crm_signal(sig, mainloop_signal_handler) == FALSE) {
	crm_signal_t *tmp = crm_signals[sig];

	crm_signals[sig] = NULL;

	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	return FALSE;
	}
	#if 0
	/* If we want signals to interrupt mainloop's poll(), instead of waiting for
	* the timeout, then we should call siginterrupt() below
	*
	* For now, just enforce a low timeout
	*/
	if (siginterrupt(sig, 1) < 0) {
	crm_perror(LOG_INFO, "Could not enable system call interruptions for signal %d", sig);
	}
	#endif

	return TRUE;
	}

	gboolean
	mainloop_destroy_signal(int sig)
	{
	crm_signal_t *tmp = NULL;

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signal(sig, NULL) == FALSE) {
	crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
	return FALSE;

	} else if (crm_signals[sig] == NULL) {
	return TRUE;
	}

	tmp = crm_signals[sig];
	crm_signals[sig] = NULL;
	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	return TRUE;
	}

	static qb_array_t *gio_map = NULL;

	/*
	* libqb...
	*/
	struct gio_to_qb_poll {
	int32_t is_used;
	GIOChannel *channel;
	int32_t events;
	void * data;
	qb_ipcs_dispatch_fn_t fn;
	enum qb_loop_priority p;
	};

	static gboolean
	gio_read_socket (GIOChannel *gio, GIOCondition condition, gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;
	gint fd = g_io_channel_unix_get_fd(gio);

	crm_trace("%p.%d %d vs. %d (G_IO_IN)", data, fd, condition, (condition & G_IO_IN));
	crm_trace("%p.%d %d vs. %d (G_IO_HUP)", data, fd, condition, (condition & G_IO_HUP));

	if(condition & G_IO_NVAL) {
	crm_trace("Marking failed adaptor %p unused", adaptor);
	adaptor->is_used = QB_FALSE;
	}

	return (adaptor->fn(fd, condition, adaptor->data) == 0);
	}

	static void
	gio_destroy(gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;
	crm_trace("Marking adaptor %p unused", adaptor);
	adaptor->is_used = QB_FALSE;
	}


	static int32_t
	gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	struct gio_to_qb_poll *adaptor;
	GIOChannel *channel;
	int32_t res = 0;

	res = qb_array_index(gio_map, fd, (void**)&adaptor);
	if (res < 0) {
	crm_err("Array lookup failed for fd=%d: %d", fd, res);
	return res;
	}

	crm_trace("Adding fd=%d to mainloop as adapater %p", fd, adaptor);
	if (adaptor->is_used) {
	crm_err("Adapter for descriptor %d is still in-use", fd);
	return -EEXIST;
	}

	channel = g_io_channel_unix_new(fd);
	if (!channel) {
	crm_err("No memory left to add fd=%d", fd);
	return -ENOMEM;
	}

	/* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
	evts \|= (G_IO_HUP\|G_IO_NVAL\|G_IO_ERR);

	adaptor->channel = channel;
	adaptor->fn = fn;
	adaptor->events = evts;
	adaptor->data = data;
	adaptor->p = p;
	adaptor->is_used = QB_TRUE;

	res = g_io_add_watch_full(channel, G_PRIORITY_DEFAULT, evts, gio_read_socket, adaptor, gio_destroy);
	crm_trace("Added to mainloop with gsource id=%d", res);
	if(res > 0) {
	return 0;
	}

	return -EINVAL;
	}

	static int32_t
	gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return 0;
	}

	static int32_t
	gio_poll_dispatch_del(int32_t fd)
	{
	struct gio_to_qb_poll *adaptor;
	crm_trace("Looking for fd=%d", fd);
	if (qb_array_index(gio_map, fd, (void**)&adaptor) == 0) {
	crm_trace("Marking adaptor %p unused", adaptor);
	if(adaptor->channel) {
	g_io_channel_unref(adaptor->channel);
	}
	adaptor->is_used = QB_FALSE;
	}
	return 0;
	}

	struct qb_ipcs_poll_handlers gio_poll_funcs = {
	.job_add = NULL,
	.dispatch_add = gio_poll_dispatch_add,
	.dispatch_mod = gio_poll_dispatch_mod,
	.dispatch_del = gio_poll_dispatch_del,
	};

	static enum qb_ipc_type
	pick_ipc_type(enum qb_ipc_type requested)
	{
	const char *env = getenv("PCMK_ipc_type");

	if(env && strcmp("shared-mem", env) == 0) {
	return QB_IPC_SHM;
	} else if(env && strcmp("socket", env) == 0) {
	return QB_IPC_SOCKET;
	} else if(env && strcmp("posix", env) == 0) {
	return QB_IPC_POSIX_MQ;
	} else if(env && strcmp("sysv", env) == 0) {
	return QB_IPC_SYSV_MQ;
	} else if(requested == QB_IPC_NATIVE) {
	/* We prefer sockets actually */
	return QB_IPC_SOCKET;
	}
	return requested;
	}

	qb_ipcs_service_t *mainloop_add_ipc_server(
	const char name, enum qb_ipc_type type, struct qb_ipcs_service_handlers callbacks)
	{
	int rc = 0;
	qb_ipcs_service_t* server = NULL;

	if(gio_map == NULL) {
	gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
	}

	server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);
	qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);

	rc = qb_ipcs_run(server);
	if (rc < 0) {
	crm_err("Could not start %s IPC server: %s (%d)", name, strerror(rc), rc);
	return NULL;
	}

	return server;
	}

	void mainloop_del_ipc_server(qb_ipcs_service_t *server)
	{
	if(server) {
	qb_ipcs_destroy(server);
	}
	}

	struct mainloop_io_s
	{
	char *name;
	void *userdata;

	guint source;
	crm_ipc_t *ipc;
	GIOChannel *channel;

	int (dispatch_fn_ipc)(const char buffer, ssize_t length, gpointer userdata);
	int (*dispatch_fn_io) (gpointer userdata);
	void (*destroy_fn) (gpointer userdata);

	};

	static gboolean
	mainloop_gio_callback(GIOChannel *gio, GIOCondition condition, gpointer data)
	{
	gboolean keep = TRUE;
	mainloop_io_t *client = data;

	crm_trace("Condition: %d %d", condition, G_IO_IN);

	if(condition & G_IO_IN) {
	if(client->ipc) {
	long rc = crm_ipc_read(client->ipc);
	crm_trace("New message from %s[%p] = %d", client->name, client, rc);

	if(rc <= 0) {
	crm_perror(LOG_TRACE, "Message acquisition failed: %ld", rc);

	} else if(client->dispatch_fn_ipc) {
	const char *buffer = crm_ipc_buffer(client->ipc);
	if(client->dispatch_fn_ipc(buffer, rc, client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}

	} else {
	crm_trace("New message from %s[%p]", client->name, client);
	if(client->dispatch_fn_io) {
	if(client->dispatch_fn_io(client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}
	}
	}

	if(client->ipc && crm_ipc_connected(client->ipc) == FALSE) {
	crm_err("Connection to %s[%p] closed", client->name, client);
	keep = FALSE;
	} else if(condition & G_IO_HUP) {
	crm_trace("Recieved G_IO_HUP for %s [%p] connection", client->name, client);
	keep = FALSE;
	} else if(condition & G_IO_NVAL) {
	crm_err("Recieved G_IO_NVAL for %s [%p] connection", client->name, client);
	keep = FALSE;
	} else if(condition & G_IO_ERR) {
	crm_err("Recieved G_IO_ERR for %s [%p] connection", client->name, client);
	keep = FALSE;
	}

	return keep;
	}

	static void
	mainloop_gio_destroy(gpointer c)
	{
	mainloop_io_t *client = c;

	crm_trace("Destroying %s[%p]", client->name, c);
	if(client->destroy_fn) {
	client->destroy_fn(client->userdata);
	}

	if(client->ipc) {
	crm_ipc_close(client->ipc);
	crm_ipc_destroy(client->ipc);
	}
	free(client->name);
	free(client);
	}

	mainloop_io_t *
	mainloop_add_ipc_client(
	- const char name, size_t max_size, void userdata, struct ipc_client_callbacks *callbacks)
	+ const char name, int priority, size_t max_size, void userdata, struct ipc_client_callbacks *callbacks)
	{
	mainloop_io_t *client = NULL;
	crm_ipc_t *conn = crm_ipc_new(name, max_size);

	if(conn && crm_ipc_connect(conn)) {
	int32_t fd = crm_ipc_get_fd(conn);
	- client = mainloop_add_fd(name, fd, userdata, NULL);
	+ client = mainloop_add_fd(name, priority, fd, userdata, NULL);
	client->ipc = conn;
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_ipc = callbacks->dispatch;
	}

	if(conn && client == NULL) {
	crm_trace("Connection to %s failed", name);
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	}

	return client;
	}

	void
	mainloop_del_ipc_client(mainloop_io_t *client)
	{
	mainloop_del_fd(client);
	}

	crm_ipc_t *
	mainloop_get_ipc_client(mainloop_io_t *client)
	{
	if(client) {
	return client->ipc;
	}
	return NULL;
	}

	mainloop_io_t *
	mainloop_add_fd(
	- const char name, int fd, void userdata, struct mainloop_fd_callbacks *callbacks)
	+ const char name, int priority, int fd, void userdata, struct mainloop_fd_callbacks *callbacks)
	{
	mainloop_io_t *client = NULL;
	if(fd > 0) {
	client = calloc(1, sizeof(mainloop_io_t));
	client->name = strdup(name);
	client->userdata = userdata;

	if(callbacks) {
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_io = callbacks->dispatch;
	}

	client->channel = g_io_channel_unix_new(fd);
	client->source = g_io_add_watch_full(
	- client->channel, G_PRIORITY_DEFAULT, (G_IO_IN\|G_IO_HUP\|G_IO_NVAL\|G_IO_ERR),
	+ client->channel, priority, (G_IO_IN\|G_IO_HUP\|G_IO_NVAL\|G_IO_ERR),
	mainloop_gio_callback, client, mainloop_gio_destroy);
	crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
	}

	return client;
	}

	void
	mainloop_del_fd(mainloop_io_t *client)
	{
	if(client != NULL) {
	if (client->channel) {
	crm_trace("Removing client %s[%p]", client->name, client);
	g_io_channel_unref(client->channel);
	client->channel = NULL;
	}
	if (client->source) {
	g_source_remove(client->source);
	client->source = 0;
	}
	/* Results in mainloop_ipcc_destroy() being called once the source is removed from mainloop? */
	}
	}

	pid_t
	mainloop_get_child_pid(mainloop_child_t *child)
	{
	return child->pid;
	}

	int
	mainloop_get_child_timeout(mainloop_child_t *child)
	{
	return child->timeout;
	}

	void *
	mainloop_get_child_userdata(mainloop_child_t *child)
	{
	return child->privatedata;
	}

	void
	mainloop_clear_child_userdata(mainloop_child_t *child)
	{
	child->privatedata = NULL;
	}

	static gboolean
	child_timeout_callback(gpointer p)
	{
	mainloop_child_t *child = p;

	child->timerid = 0;
	if (child->timeout) {
	crm_crit("%s process (PID %d) will not die!", child->desc, (int)child->pid);
	return FALSE;
	}

	child->timeout = TRUE;
	crm_warn("%s process (PID %d) timed out", child->desc, (int)child->pid);

	if (kill(child->pid, SIGKILL) < 0) {
	if (errno == ESRCH) {
	/* Nothing left to do */
	return FALSE;
	}
	crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
	}

	child->timerid = g_timeout_add(5000, child_timeout_callback, child);
	return FALSE;
	}

	static void
	mainloop_child_destroy(mainloop_child_t *child)
	{
	if (child->timerid != 0) {
	crm_trace("Removing timer %d", child->timerid);
	g_source_remove(child->timerid);
	child->timerid = 0;
	}

	free(child->desc);
	g_free(child);
	}

	static void
	child_death_dispatch(GPid pid, gint status, gpointer user_data)
	{
	int signo = 0;
	int exitcode = 0;
	mainloop_child_t *child = user_data;

	crm_trace("Managed process %d exited: %p", pid, child);

	if (WIFEXITED(status)) {
	exitcode = WEXITSTATUS(status);
	crm_trace("Managed process %d (%s) exited with rc=%d", pid,
	child->desc, exitcode);

	} else if (WIFSIGNALED(status)) {
	signo = WTERMSIG(status);
	crm_trace("Managed process %d (%s) exited with signal=%d", pid,
	child->desc, signo);
	}
	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	crm_err("Managed process %d (%s) dumped core", pid, child->desc);
	}
	#endif

	if (child->callback) {
	child->callback(child, status, signo, exitcode);
	}
	crm_trace("Removed process entry for %d", pid);

	mainloop_child_destroy(child);
	return;
	}

	/* Create/Log a new tracked process
	* To track a process group, use -pid
	*/
	void
	mainloop_add_child(pid_t pid, int timeout, const char desc, void privatedata,
	void (callback)(mainloop_child_t p, int status, int signo, int exitcode))
	{
	mainloop_child_t *child = g_new(mainloop_child_t, 1);

	child->pid = pid;
	child->timerid = 0;
	child->timeout = FALSE;
	child->desc = strdup(desc);
	child->privatedata = privatedata;
	child->callback = callback;

	if (timeout) {
	child->timerid = g_timeout_add(
	timeout, child_timeout_callback, child);
	}

	child->watchid = g_child_watch_add(pid, child_death_dispatch, child);
	}
	diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
	index 04976f666f..b6248629ea 100644
	--- a/lib/fencing/st_client.c
	+++ b/lib/fencing/st_client.c
	@@ -1,2018 +1,2018 @@
	/*
	* Copyright (c) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/stat.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>
	#include <libgen.h> /* Add it for compiling on OSX */

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#ifdef HAVE_STONITH_STONITH_H
	# include <stonith/stonith.h>
	# define LHA_STONITH_LIBRARY "libstonith.so.1"
	static void *lha_agents_lib = NULL;
	#endif

	#include <crm/common/mainloop.h>


	CRM_TRACE_INIT_DATA(stonith);

	typedef struct stonith_private_s {
	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;
	GHashTable *stonith_op_callback_table;
	GList *notify_list;

	void (op_callback) (stonith_t st, const xmlNode * msg, int call, int rc, xmlNode * output,
	void *userdata);

	} stonith_private_t;

	typedef struct stonith_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (notify) (stonith_t st, stonith_event_t *e);

	} stonith_notify_client_t;

	typedef struct stonith_callback_client_s {
	void (callback) (stonith_t st, const xmlNode * msg, int call, int rc, xmlNode * output,
	void *userdata);
	const char *id;
	void *user_data;
	gboolean only_success;
	struct timer_rec_s *timer;

	} stonith_callback_client_t;

	struct notify_blob_s {
	stonith_t *stonith;
	xmlNode *xml;
	};

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	stonith_t *stonith;
	};

	typedef int (stonith_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	static const char META_TEMPLATE[] =
	"<?xml version=\"1.0\"?>\n"
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
	"<resource-agent name=\"%s\">\n"
	" <version>1.0</version>\n"
	" <longdesc lang=\"en\">\n"
	"%s\n"
	" </longdesc>\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	"%s\n"
	" <actions>\n"
	" <action name=\"start\" timeout=\"20\" />\n"
	" <action name=\"stop\" timeout=\"15\" />\n"
	" <action name=\"status\" timeout=\"20\" />\n"
	" <action name=\"monitor\" timeout=\"20\" interval=\"3600\"/>\n"
	" <action name=\"meta-data\" timeout=\"15\" />\n"
	" </actions>\n"
	" <special tag=\"heartbeat\">\n"
	" <version>2.0</version>\n" " </special>\n" "</resource-agent>\n";

	bool stonith_dispatch(stonith_t * st);
	int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata);
	void stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc);
	xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);
	int stonith_send_command(stonith_t * stonith, const char op, xmlNode data,
	xmlNode ** output_data, int call_options, int timeout);

	static void stonith_connection_destroy(gpointer user_data);
	static void stonith_send_notification(gpointer data, gpointer user_data);

	static void
	stonith_connection_destroy(gpointer user_data)
	{
	stonith_t *stonith = user_data;
	stonith_private_t *native = NULL;
	struct notify_blob_s blob;

	crm_trace("Sending destroyed notification");
	blob.stonith = stonith;
	blob.xml = create_xml_node(NULL, "notify");

	native = stonith->private;
	native->ipc = NULL;
	native->source = NULL;

	stonith->state = stonith_disconnected;
	crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT);

	g_list_foreach(native->notify_list, stonith_send_notification, &blob);
	free_xml(blob.xml);
	}

	xmlNode *
	create_device_registration_xml(const char id, const char namespace, const char *agent,
	stonith_key_value_t * params)
	{
	xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);

	crm_xml_add(data, XML_ATTR_ID, id);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, "agent", agent);
	crm_xml_add(data, "namespace", namespace);

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	return data;
	}

	static int
	stonith_api_register_device(stonith_t * st, int call_options,
	const char id, const char namespace, const char *agent,
	stonith_key_value_t * params)
	{
	int rc = 0;
	xmlNode *data = NULL;

	#if HAVE_STONITH_STONITH_H
	namespace = get_stonith_provider(agent, namespace);
	if (strcmp(namespace, "heartbeat") == 0) {
	stonith_key_value_add(params, "plugin", agent);
	agent = "fence_legacy";
	}
	#endif

	data = create_device_registration_xml(id, namespace, agent, params);

	rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, XML_ATTR_ID, name);
	rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_LEVEL);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add_int(data, XML_ATTR_ID, level);
	rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	xmlNode *
	create_level_registration_xml(const char node, int level, stonith_key_value_t device_list)
	{
	xmlNode *data = create_xml_node(NULL, F_STONITH_LEVEL);

	crm_xml_add_int(data, XML_ATTR_ID, level);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add(data, "origin", __FUNCTION__);

	for (; device_list; device_list = device_list->next) {
	xmlNode *dev = create_xml_node(data, F_STONITH_DEVICE);

	crm_xml_add(dev, XML_ATTR_ID, device_list->value);
	}

	return data;
	}

	static int
	stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
	stonith_key_value_t * device_list)
	{
	int rc = 0;
	xmlNode *data = create_level_registration_xml(node, level, device_list);

	rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	static void
	append_arg(gpointer key, gpointer value, gpointer user_data)
	{
	int len = 3; /* =, \n, \0 */
	int last = 0;
	char **args = user_data;

	CRM_CHECK(key != NULL, return);
	CRM_CHECK(value != NULL, return);

	if (strstr(key, "pcmk_")) {
	return;
	} else if (strstr(key, CRM_META)) {
	return;
	} else if (safe_str_eq(key, "crm_feature_set")) {
	return;
	}

	len += strlen(key);
	len += strlen(value);
	if (*args != NULL) {
	last = strlen(*args);
	}

	args = realloc(args, last + len);
	crm_trace("Appending: %s=%s", (char )key, (char )value);
	sprintf((args) + last, "%s=%s\n", (char )key, (char *)value);
	}

	static void
	append_const_arg(const char key, const char value, char **arg_list)
	{
	char *glib_sucks_key = strdup(key);
	char *glib_sucks_value = strdup(value);

	append_arg(glib_sucks_key, glib_sucks_value, arg_list);

	free(glib_sucks_value);
	free(glib_sucks_key);
	}

	static void
	append_host_specific_args(const char victim, const char map, GHashTable * params, char **arg_list)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0;

	if (map == NULL) {
	/* The best default there is for now... */
	crm_debug("Using default arg map: port=uname");
	append_const_arg("port", victim, arg_list);
	return;
	}

	max = strlen(map);
	crm_debug("Processing arg map: %s", map);
	for (; lpc < max + 1; lpc++) {
	if (isalpha(map[lpc])) {
	/* keep going */

	} else if (map[lpc] == '=' \|\| map[lpc] == ':') {
	free(name);
	name = calloc(1, 1 + lpc - last);
	memcpy(name, map + last, lpc - last);
	crm_debug("Got name: %s", name);
	last = lpc + 1;

	} else if (map[lpc] == 0 \|\| map[lpc] == ',' \|\| isspace(map[lpc])) {
	char *param = NULL;
	const char *value = NULL;

	param = calloc(1, 1 + lpc - last);
	memcpy(param, map + last, lpc - last);
	last = lpc + 1;

	crm_debug("Got key: %s", param);
	if (name == NULL) {
	crm_err("Misparsed '%s', found '%s' without a name", map, param);
	free(param);
	continue;
	}

	if (safe_str_eq(param, "uname")) {
	value = victim;
	} else {
	char *key = crm_meta_name(param);

	value = g_hash_table_lookup(params, key);
	free(key);
	}

	if (value) {
	crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim);
	append_const_arg(name, value, arg_list);

	} else {
	crm_err("No node attribute '%s' for '%s'", name, victim);
	}

	free(name);
	name = NULL;
	free(param);
	if (map[lpc] == 0) {
	break;
	}

	} else if (isspace(map[lpc])) {
	last = lpc;
	}
	}
	free(name);
	}

	static char *
	make_args(const char action, const char victim, GHashTable * device_args, GHashTable * port_map)
	{
	char buffer[512];
	char *arg_list = NULL;
	const char *value = NULL;

	CRM_CHECK(action != NULL, return NULL);

	if (device_args) {
	g_hash_table_foreach(device_args, append_arg, &arg_list);
	}

	buffer[511] = 0;
	snprintf(buffer, 511, "pcmk_%s_action", action);
	if (device_args) {
	value = g_hash_table_lookup(device_args, buffer);
	}

	if (value == NULL && device_args) {
	/* Legacy support for early 1.1 releases - Remove for 1.2 */
	snprintf(buffer, 511, "pcmk_%s_cmd", action);
	value = g_hash_table_lookup(device_args, buffer);
	}

	if (value) {
	crm_info("Substituting action '%s' for requested operation '%s'", value, action);
	action = value;
	}

	append_const_arg(STONITH_ATTR_ACTION_OP, action, &arg_list);
	if (victim && device_args) {
	const char *alias = victim;
	const char *param = g_hash_table_lookup(device_args, STONITH_ATTR_HOSTARG);

	if (port_map && g_hash_table_lookup(port_map, victim)) {
	alias = g_hash_table_lookup(port_map, victim);
	}

	/* Always supply the node's name too:
	* https://fedorahosted.org/cluster/wiki/FenceAgentAPI
	*/
	append_const_arg("nodename", victim, &arg_list);

	/* Check if we need to supply the victim in any other form */
	if (param == NULL) {
	const char *map = g_hash_table_lookup(device_args, STONITH_ATTR_ARGMAP);

	if (map == NULL) {
	param = "port";
	value = g_hash_table_lookup(device_args, param);

	} else {
	/* Legacy handling */
	append_host_specific_args(alias, map, device_args, &arg_list);
	value = map; /* Nothing more to do */
	}

	} else if (safe_str_eq(param, "none")) {
	value = param; /* Nothing more to do */

	} else {
	value = g_hash_table_lookup(device_args, param);
	}

	/* Don't overwrite explictly set values for $param */
	if (value == NULL \|\| safe_str_eq(value, "dynamic")) {
	crm_debug("Performing %s action for node '%s' as '%s=%s'", action, victim, param,
	alias);
	append_const_arg(param, alias, &arg_list);
	}
	}

	crm_trace("Calculated: %s", arg_list);
	return arg_list;
	}

	static gboolean
	st_child_term(gpointer data)
	{
	int rc = 0;
	async_command_t * track = data;
	crm_info("Child %d timed out, sending SIGTERM", track->pid);
	track->timer_sigterm = 0;
	rc = kill(track->pid, SIGTERM);
	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't send SIGTERM to %d", track->pid);
	}
	return FALSE;
	}

	static gboolean
	st_child_kill(gpointer data)
	{
	int rc = 0;
	async_command_t * track = data;
	crm_info("Child %d timed out, sending SIGKILL", track->pid);
	track->timer_sigkill = 0;
	rc = kill(track->pid, SIGKILL);
	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't send SIGKILL to %d", track->pid);
	}
	return FALSE;
	}

	/* Borrowed from libfence and extended */
	int
	run_stonith_agent(const char agent, const char action, const char *victim,
	GHashTable * device_args, GHashTable * port_map, int agent_result, char *output,
	async_command_t * track)
	{
	char *args = make_args(action, victim, device_args, port_map);
	int pid, status, len, rc = -EPROTO;
	int p_read_fd, p_write_fd; /* parent read/write file descriptors */
	int c_read_fd, c_write_fd; /* child read/write file descriptors */
	int fd1[2];
	int fd2[2];

	c_read_fd = c_write_fd = p_read_fd = p_write_fd = -1;

	if (args == NULL \|\| agent == NULL)
	goto fail;
	len = strlen(args);

	if (pipe(fd1))
	goto fail;
	p_read_fd = fd1[0];
	c_write_fd = fd1[1];

	if (pipe(fd2))
	goto fail;
	c_read_fd = fd2[0];
	p_write_fd = fd2[1];

	crm_debug("forking");
	pid = fork();
	if (pid < 0) {
	rc = -ECHILD;
	goto fail;
	}

	if (pid) {
	/* parent */
	int ret;
	int total = 0;

	fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) \| O_NONBLOCK);

	do {
	crm_debug("sending args");
	ret = write(p_write_fd, args + total, len - total);
	if (ret > 0) {
	total += ret;
	}

	} while (errno == EINTR && total < len);

	if (total != len) {
	crm_perror(LOG_ERR, "Sent %d not %d bytes", total, len);
	if (ret >= 0) {
	rc = -EREMOTEIO;
	}
	goto fail;
	}

	close(p_write_fd);

	if (track && track->done) {
	track->stdout = p_read_fd;
	g_child_watch_add(pid, track->done, track);
	crm_trace("Op: %s on %s, pid: %d, timeout: %ds", action, agent, pid, track->timeout);

	if (track->timeout) {
	track->pid = pid;
	track->timer_sigterm = g_timeout_add(1000*track->timeout, st_child_term, track);
	track->timer_sigkill = g_timeout_add(1000*(track->timeout+5), st_child_kill, track);

	} else {
	crm_err("No timeout set for stonith operation %s with device %s", action, agent);
	}

	close(c_write_fd);
	close(c_read_fd);
	free(args);
	return pid;

	} else {
	pid_t p;

	do {
	p = waitpid(pid, &status, 0);
	} while (p < 0 && errno == EINTR);

	if (p < 0) {
	crm_perror(LOG_ERR, "waitpid(%d)", pid);

	} else if (p != pid) {
	crm_err("Waited for %d, got %d", pid, p);
	}

	if (output != NULL) {
	char *local_copy;
	int lpc = 0, last = 0, more;
	len = 0;
	do {
	char buf[500];

	ret = read(p_read_fd, buf, 500);
	if (ret > 0) {
	buf[ret] = 0;
	output = realloc(output, len + ret + 1);
	sprintf((*output) + len, "%s", buf);
	crm_trace("%d: %s", ret, (*output) + len);
	len += ret;
	}

	} while (ret == 500 \|\| (ret < 0 && errno == EINTR));

	if (*output) {
	local_copy = strdup(*output);
	more = strlen(local_copy);
	for(lpc = 0; lpc < more; lpc++) {
	if(local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	crm_debug("%s: %s", agent, local_copy+last);
	last = lpc+1;
	}
	}
	crm_debug("%s: %s (total %d bytes)", agent, local_copy+last, more);
	free(local_copy);
	}
	}

	rc = -ECONNABORTED;
	*agent_result = -ECONNABORTED;
	if (WIFEXITED(status)) {
	crm_debug("result = %d", WEXITSTATUS(status));
	*agent_result = -WEXITSTATUS(status);
	rc = 0;

	} else if (WIFSIGNALED(status)) {
	crm_err("call %s for %s exited due to signal %d", action, agent, WTERMSIG(status));

	} else {
	crm_err("call %s for %s exited abnormally. stopped=%d, continued=%d",
	action, agent, WIFSTOPPED(status), WIFCONTINUED(status));
	}
	}

	} else {
	/* child */

	close(1);
	/* coverity[leaked_handle] False positive */
	if (dup(c_write_fd) < 0)
	goto fail;
	close(2);
	/* coverity[leaked_handle] False positive */
	if (dup(c_write_fd) < 0)
	goto fail;
	close(0);
	/* coverity[leaked_handle] False positive */
	if (dup(c_read_fd) < 0)
	goto fail;

	/* keep c_write_fd open so parent can report all errors. */
	close(c_read_fd);
	close(p_read_fd);
	close(p_write_fd);

	execlp(agent, agent, NULL);
	exit(EXIT_FAILURE);
	}

	fail:
	free(args);

	if (p_read_fd >= 0) {
	close(p_read_fd);
	}
	if (p_write_fd >= 0) {
	close(p_write_fd);
	}

	if (c_read_fd >= 0) {
	close(c_read_fd);
	}
	if (c_write_fd >= 0) {
	close(c_write_fd);
	}

	return rc;
	}

	static int
	stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace,
	stonith_key_value_t ** devices, int timeout)
	{
	int count = 0;

	if (devices == NULL) {
	crm_err("Parameter error: stonith_api_device_list");
	return -EFAULT;
	}

	/* Include Heartbeat agents */
	if (namespace == NULL \|\| safe_str_eq("heartbeat", namespace)) {
	#if HAVE_STONITH_STONITH_H
	static gboolean need_init = TRUE;

	char **entry = NULL;
	char **type_list = NULL;
	static char *(type_list_fn) (void) = NULL;
	static void (type_free_fn) (char *) = NULL;

	if(need_init) {
	need_init = FALSE;
	type_list_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_types", FALSE);
	type_free_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_free_hostlist", FALSE);
	}

	if(type_list_fn) {
	type_list = (*type_list_fn)();
	}

	for (entry = type_list; entry != NULL && *entry; ++entry) {
	crm_trace("Added: %s", *entry);
	devices = stonith_key_value_add(devices, NULL, *entry);
	count++;
	}
	if (type_list && type_free_fn) {
	(*type_free_fn)(type_list);
	}
	#else
	if(namespace != NULL) {
	return -EINVAL; /* Heartbeat agents not supported */
	}
	#endif
	}

	/* Include Red Hat agents, basically: ls -1 @sbin_dir@/fence_* */
	if (namespace == NULL \|\| safe_str_eq("redhat", namespace)) {
	struct dirent **namelist;
	int file_num = scandir(RH_STONITH_DIR, &namelist, 0, alphasort);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX + 1];

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (0 != strncmp(RH_STONITH_PREFIX,
	namelist[file_num]->d_name, strlen(RH_STONITH_PREFIX))) {
	free(namelist[file_num]);
	continue;
	}

	snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	devices = stonith_key_value_add(devices, NULL, namelist[file_num]->d_name);
	count++;
	}

	free(namelist[file_num]);
	}
	free(namelist);
	}
	}

	return count;
	}

	static int
	stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent,
	const char namespace, char *output, int timeout)
	{
	int rc = 0;
	char *buffer = NULL;
	const char *provider = get_stonith_provider(agent, namespace);

	crm_trace("looking up %s/%s metadata", agent, provider);

	/* By having this in a library, we can access it from stonith_admin
	* when neither lrmd or stonith-ng are running
	* Important for the crm shell's validations...
	*/

	if (safe_str_eq(provider, "redhat")) {

	int exec_rc = run_stonith_agent(agent, "metadata", NULL, NULL, NULL, &rc, &buffer, NULL);

	if (exec_rc < 0 \|\| rc != 0 \|\| buffer == NULL) {
	crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer));
	free(buffer); /* Just in case */
	return -EINVAL;

	} else {

	xmlNode *xml = string2xml(buffer);
	xmlNode *actions = NULL;
	xmlXPathObject *xpathObj = NULL;

	xpathObj = xpath_search(xml, "//actions");
	if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
	actions = getXpathResult(xpathObj, 0);
	}

	/* Now fudge the metadata so that the start/stop actions appear */
	xpathObj = xpath_search(xml, "//action[@name='stop']");
	if (xpathObj == NULL \|\| xpathObj->nodesetval->nodeNr <= 0) {
	xmlNode *tmp = NULL;

	tmp = create_xml_node(actions, "action");
	crm_xml_add(tmp, "name", "stop");
	crm_xml_add(tmp, "timeout", "20s");

	tmp = create_xml_node(actions, "action");
	crm_xml_add(tmp, "name", "start");
	crm_xml_add(tmp, "timeout", "20s");
	}

	/* Now fudge the metadata so that the port isn't required in the configuration */
	xpathObj = xpath_search(xml, "//parameter[@name='port']");
	if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
	/* We'll fill this in */
	xmlNode *tmp = getXpathResult(xpathObj, 0);

	crm_xml_add(tmp, "required", "0");
	}

	free(buffer);
	buffer = dump_xml_formatted(xml);
	free_xml(xml);
	}

	} else {
	#if !HAVE_STONITH_STONITH_H
	return -EINVAL; /* Heartbeat agents not supported */
	#else
	int bufferlen = 0;

	char *xml_meta_longdesc = NULL;
	char *xml_meta_shortdesc = NULL;

	char *meta_param = NULL;
	char *meta_longdesc = NULL;
	char *meta_shortdesc = NULL;
	static const char *no_parameter_info = "<!-- no value -->";

	Stonith *stonith_obj = NULL;

	static gboolean need_init = TRUE;
	static Stonith (st_new_fn) (const char *) = NULL;
	static const char (st_info_fn) (Stonith *, int) = NULL;
	static void (st_del_fn) (Stonith ) = NULL;

	if(need_init) {
	need_init = FALSE;
	st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE);
	st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE);
	st_info_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_get_info", FALSE);
	}

	if (lha_agents_lib && st_new_fn && st_del_fn && st_info_fn) {
	stonith_obj = (*st_new_fn) (agent);
	if(stonith_obj) {
	meta_longdesc = strdup((*st_info_fn)(stonith_obj, ST_DEVICEDESCR));
	if (meta_longdesc == NULL) {
	crm_warn("no long description in %s's metadata.", agent);
	meta_longdesc = strdup(no_parameter_info);
	}

	meta_shortdesc = strdup((*st_info_fn)(stonith_obj, ST_DEVICEID));
	if (meta_shortdesc == NULL) {
	crm_warn("no short description in %s's metadata.", agent);
	meta_shortdesc = strdup(no_parameter_info);
	}

	meta_param = strdup((*st_info_fn)(stonith_obj, ST_CONF_XML));
	if (meta_param == NULL) {
	crm_warn("no list of parameters in %s's metadata.", agent);
	meta_param = strdup(no_parameter_info);
	}
	(*st_del_fn)(stonith_obj);
	} else {
	return -EINVAL; /* Heartbeat agents not supported */
	}
	}

	xml_meta_longdesc =
	(char )xmlEncodeEntitiesReentrant(NULL, (const unsigned char )meta_longdesc);
	xml_meta_shortdesc =
	(char )xmlEncodeEntitiesReentrant(NULL, (const unsigned char )meta_shortdesc);

	bufferlen = strlen(META_TEMPLATE) + strlen(agent)
	+ strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc)
	+ strlen(meta_param) + 1;

	buffer = calloc(1, bufferlen);
	snprintf(buffer, bufferlen - 1, META_TEMPLATE,
	agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param);

	xmlFree(xml_meta_longdesc);
	xmlFree(xml_meta_shortdesc);

	free(meta_shortdesc);
	free(meta_longdesc);
	free(meta_param);
	#endif
	}

	if (output) {
	*output = buffer;

	} else {
	free(buffer);
	}

	return rc;
	}

	static int
	stonith_api_query(stonith_t * stonith, int call_options, const char *target,
	stonith_key_value_t ** devices, int timeout)
	{
	int rc = 0, lpc = 0, max = 0;

	xmlNode *data = NULL;
	xmlNode *output = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	CRM_CHECK(devices != NULL, return -EINVAL);

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, target);
	rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);

	if (rc < 0) {
	return rc;
	}

	xpathObj = xpath_search(output, "//@agent");
	if (xpathObj) {
	max = xpathObj->nodesetval->nodeNr;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_CHECK(match != NULL, continue);

	crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match));
	devices = stonith_key_value_add(devices, NULL, crm_element_value(match, XML_ATTR_ID));
	}
	}

	free_xml(output);
	free_xml(data);
	return max;
	}

	static int
	stonith_api_call(stonith_t * stonith,
	int call_options,
	const char *id,
	const char *action,
	const char *victim,
	int timeout,
	xmlNode **output)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, F_STONITH_DEVICE);
	crm_xml_add(data, "origin", __FUNCTION__);
	crm_xml_add(data, F_STONITH_DEVICE, id);
	crm_xml_add(data, F_STONITH_ACTION, action);
	crm_xml_add(data, F_STONITH_TARGET, victim);

	rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_list(stonith_t * stonith, int call_options, const char id, char *list_info, int timeout)
	{
	int rc;
	xmlNode *output = NULL;

	rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output);

	if (output && list_info) {
	const char *list_str;

	list_str = crm_element_value(output, "st_output");

	if (list_str) {
	*list_info = strdup(list_str);
	}
	}

	if (output) {
	free_xml(output);
	}

	return rc;
	}

	static int
	stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
	{
	return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL);
	}

	static int
	stonith_api_status(stonith_t * stonith, int call_options, const char id, const char port, int timeout)
	{
	return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL);
	}

	static int
	stonith_api_fence(stonith_t * stonith, int call_options, const char node, const char action,
	int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	crm_xml_add(data, F_STONITH_ACTION, action);
	crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout);

	rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
	{
	return stonith_api_fence(stonith, call_options \| st_opt_manual_ack, target, "off", 0);
	}

	static int
	stonith_api_history(stonith_t * stonith, int call_options, const char *node,
	stonith_history_t ** history, int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;
	xmlNode *output = NULL;
	stonith_history_t *last = NULL;

	*history = NULL;

	if (node) {
	data = create_xml_node(NULL, __FUNCTION__);
	crm_xml_add(data, F_STONITH_TARGET, node);
	}

	rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
	call_options \| st_opt_sync_call, timeout);
	free_xml(data);

	if (rc == 0) {
	xmlNode *op = NULL;
	xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output, LOG_ERR);

	for (op = __xml_first_child(reply); op != NULL; op = __xml_next(op)) {
	stonith_history_t *kvp;

	kvp = calloc(1, sizeof(stonith_history_t));
	kvp->target = crm_element_value_copy(op, F_STONITH_TARGET);
	kvp->action = crm_element_value_copy(op, F_STONITH_ACTION);
	kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN);
	kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE);
	crm_element_value_int(op, F_STONITH_DATE, &kvp->completed);
	crm_element_value_int(op, F_STONITH_STATE, &kvp->state);

	if (last) {
	last->next = kvp;
	} else {
	*history = kvp;
	}
	last = kvp;
	}
	}
	return rc;
	}

	gboolean
	is_redhat_agent(const char *agent)
	{
	int rc = 0;
	struct stat prop;
	char buffer[FILENAME_MAX + 1];

	snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, agent);
	rc = stat(buffer, &prop);
	if (rc >= 0 && S_ISREG(prop.st_mode)) {
	return TRUE;
	}
	return FALSE;
	}

	const char *
	get_stonith_provider(const char agent, const char provider)
	{
	/* This function sucks */
	if (is_redhat_agent(agent)) {
	return "redhat";

	#if HAVE_STONITH_STONITH_H
	} else {
	Stonith *stonith_obj = NULL;

	static gboolean need_init = TRUE;
	static Stonith (st_new_fn) (const char *) = NULL;
	static void (st_del_fn) (Stonith ) = NULL;

	if(need_init) {
	need_init = FALSE;
	st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE);
	st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE);
	}

	if (lha_agents_lib && st_new_fn && st_del_fn) {
	stonith_obj = (*st_new_fn) (agent);
	if(stonith_obj) {
	(*st_del_fn)(stonith_obj);
	return "heartbeat";
	}
	}
	#endif
	}

	crm_err("No such device: %s", agent);
	return NULL;
	}

	static gint
	stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const stonith_notify_client_t *a_client = a;
	const stonith_notify_client_t *b_client = b;

	CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
	rc = strcmp(a_client->event, b_client->event);
	if (rc == 0) {
	if (a_client->notify == NULL \|\| b_client->notify == NULL) {
	return 0;

	} else if (a_client->notify == b_client->notify) {
	return 0;

	} else if (((long)a_client->notify) < ((long)b_client->notify)) {
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return -1;
	}
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return 1;
	}
	return rc;
	}

	xmlNode *
	stonith_create_op(int call_id, const char token, const char op, xmlNode * data, int call_options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "stonith_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command");

	crm_xml_add(op_msg, F_TYPE, T_STONITH_NG);
	crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_STONITH_OPERATION, op);
	crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, F_STONITH_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	stonith_destroy_op_callback(gpointer data)
	{
	stonith_callback_client_t *blob = data;

	if (blob->timer && blob->timer->ref > 0) {
	g_source_remove(blob->timer->ref);
	}
	free(blob->timer);
	free(blob);
	}

	static int
	stonith_api_signoff(stonith_t * stonith)
	{
	stonith_private_t *native = stonith->private;

	crm_debug("Signing out of the STONITH Service");

	if (native->ipc != NULL) {
	/* If attached to mainloop and it is active, _close() will result in:
	* - the source being removed from mainloop
	* - the dnotify callback being invoked
	* Otherwise, we are at least correctly disconnecting IPC
	*/
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	stonith->state = stonith_disconnected;
	return pcmk_ok;
	}

	static int
	stonith_api_signon(stonith_t * stonith, const char name, int stonith_fd)
	{
	int rc = pcmk_ok;
	stonith_private_t *native = stonith->private;
	static struct ipc_client_callbacks st_callbacks =
	{
	.dispatch = stonith_dispatch_internal,
	.destroy = stonith_connection_destroy
	};

	crm_trace("Connecting command channel");

	stonith->state = stonith_connected_command;
	if(stonith_fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("stonith-ng", 0);

	if(native->ipc && crm_ipc_connect(native->ipc)) {
	*stonith_fd = crm_ipc_get_fd(native->ipc);

	} else if(native->ipc) {
	rc = -ENOTCONN;
	}

	} else {
	/* With mainloop */
	- native->source = mainloop_add_ipc_client("stonith-ng", 0, stonith, &st_callbacks);
	+ native->source = mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the Stonith API");
	rc = -ENOTCONN;
	}

	if (rc == pcmk_ok) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "stonith_command");

	crm_xml_add(hello, F_TYPE, T_STONITH_NG);
	crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_STONITH_CLIENTNAME, name);
	rc = crm_ipc_send(native->ipc, hello, &reply, -1);

	if(rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the fencing API: %d", rc);
	rc = -ECOMM;

	} else if(reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;

	} else {
	const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_STONITH_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;

	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;

	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	#if HAVE_MSGFROMIPC_TIMEOUT
	stonith->call_timeout = MAX_IPC_DELAY;
	#endif
	crm_debug("Connection to STONITH successful");
	return pcmk_ok;
	}

	crm_debug("Connection to STONITH failed: %s", pcmk_strerror(rc));
	stonith->cmds->disconnect(stonith);
	return rc;
	}

	static int
	stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
	{
	xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__);
	stonith_private_t *native = stonith->private;

	if (stonith->state != stonith_disconnected) {
	int rc;

	crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY);
	if (enabled) {
	crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback);
	} else {
	crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback);
	}
	rc = crm_ipc_send(native->ipc, notify_msg, NULL, -1);
	if(rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
	rc = -ECOMM;
	}
	}

	free_xml(notify_msg);
	return pcmk_ok;
	}

	static int
	stonith_api_add_notification(stonith_t * stonith, const char *event,
	void (callback) (stonith_t stonith, stonith_event_t *e))
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	private = stonith->private;
	crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));

	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = callback;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	if (list_item != NULL) {
	crm_warn("Callback already present");
	free(new_client);
	return -ENOTUNIQ;

	} else {
	private->notify_list = g_list_append(private->notify_list, new_client);

	stonith_set_notification(stonith, event, 1);

	crm_trace("Callback added (%d)", g_list_length(private->notify_list));
	}
	return pcmk_ok;
	}

	static int
	stonith_api_del_notification(stonith_t * stonith, const char *event)
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	crm_debug("Removing callback for %s events", event);

	private = stonith->private;
	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = NULL;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	stonith_set_notification(stonith, event, 0);

	if (list_item != NULL) {
	stonith_notify_client_t *list_client = list_item->data;

	private->notify_list = g_list_remove(private->notify_list, list_client);
	free(list_client);

	crm_trace("Removed callback");

	} else {
	crm_trace("Callback not present");
	}
	free(new_client);
	return pcmk_ok;
	}

	static gboolean
	stonith_async_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
	stonith_perform_callback(timer->stonith, NULL, timer->call_id, -ETIME);

	/* Always return TRUE, never remove the handler
	* We do that in stonith_del_callback()
	*/
	return TRUE;
	}

	static int
	stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, bool only_success,
	void user_data, const char callback_name,
	void (callback) (stonith_t st, const xmlNode * msg, int call, int rc,
	xmlNode * output, void *userdata))
	{
	stonith_callback_client_t *blob = NULL;
	stonith_private_t *private = NULL;

	CRM_CHECK(stonith != NULL, return -EINVAL);
	CRM_CHECK(stonith->private != NULL, return -EINVAL);
	private = stonith->private;

	if (call_id == 0) {
	private->op_callback = callback;

	} else if (call_id < 0) {
	if (only_success == FALSE) {
	crm_trace("Call failed, calling %s: %s",
	callback_name, pcmk_strerror(call_id));
	callback(stonith, NULL, call_id, call_id, NULL, user_data);
	} else {
	crm_warn("STONITH call failed: %s", pcmk_strerror(call_id));
	}
	return FALSE;
	}

	blob = calloc(1, sizeof(stonith_callback_client_t));
	blob->id = callback_name;
	blob->only_success = only_success;
	blob->user_data = user_data;
	blob->callback = callback;

	if (timeout > 0) {
	struct timer_rec_s *async_timer = NULL;

	async_timer = calloc(1, sizeof(struct timer_rec_s));
	blob->timer = async_timer;

	async_timer->stonith = stonith;
	async_timer->call_id = call_id;
	/* Allow a fair bit of grace to allow the server to tell us of a timeout
	* This is only a fallback
	*/
	async_timer->timeout = (timeout + 60) * 1000;
	async_timer->ref =
	g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
	}

	g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob);
	crm_trace("Added callback to %s for call %d", callback_name, call_id);

	return TRUE;
	}

	static int
	stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
	{
	stonith_private_t *private = stonith->private;

	if (all_callbacks) {
	private->op_callback = NULL;
	g_hash_table_destroy(private->stonith_op_callback_table);
	private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL,
	stonith_destroy_op_callback);

	} else if (call_id == 0) {
	private->op_callback = NULL;

	} else {
	g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id));
	}
	return pcmk_ok;
	}

	static void
	stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	int call = GPOINTER_TO_INT(key);
	stonith_callback_client_t *blob = value;

	crm_debug("Call %d (%s): pending", call, crm_str(blob->id));
	}

	void
	stonith_dump_pending_callbacks(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->private;

	if (private->stonith_op_callback_table == NULL) {
	return;
	}
	return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
	}

	void
	stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	stonith_private_t *private = NULL;
	stonith_callback_client_t *blob = NULL;
	stonith_callback_client_t local_blob;

	CRM_CHECK(stonith != NULL, return);
	CRM_CHECK(stonith->private != NULL, return);

	private = stonith->private;

	local_blob.id = NULL;
	local_blob.callback = NULL;
	local_blob.user_data = NULL;
	local_blob.only_success = FALSE;

	if (msg != NULL) {
	crm_element_value_int(msg, F_STONITH_RC, &rc);
	crm_element_value_int(msg, F_STONITH_CALLID, &call_id);
	output = get_message_xml(msg, F_STONITH_CALLDATA);
	}

	CRM_CHECK(call_id > 0, crm_log_xml_err(msg, "Bad result"));

	blob = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id));

	if (blob != NULL) {
	local_blob = *blob;
	blob = NULL;

	stonith_api_del_callback(stonith, call_id, FALSE);

	} else {
	crm_trace("No callback found for call %d", call_id);
	local_blob.callback = NULL;
	}

	if (local_blob.callback != NULL && (rc == pcmk_ok \|\| local_blob.only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
	local_blob.callback(stonith, msg, call_id, rc, output, local_blob.user_data);

	} else if (private->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("STONITH command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed STONITH Update");
	}

	if (private->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	private->op_callback(stonith, msg, call_id, rc, output, NULL);
	}
	crm_trace("OP callback activated.");
	}

	/*
	<notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
	<st_calldata >
	<stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="stonith-test" >
	<st_calldata >
	<st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
	<attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
	</st_device_id>
	</st_calldata>
	</stonith_command>
	</st_calldata>
	</notify>

	<notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
	<st_calldata >
	<st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
	</st_calldata>
	</notify>
	*/
	static stonith_event_t *
	xml_to_event(xmlNode *msg)
	{
	stonith_event_t *event = calloc(1, sizeof(stonith_event_t));
	const char *ntype = crm_element_value(msg, F_SUBTYPE);
	char *data_addr = g_strdup_printf("//%s", ntype);
	xmlNode *data = get_xpath_object(data_addr, msg, LOG_ERR);

	crm_log_xml_trace(msg, "stonith_notify");

	crm_element_value_int(msg, F_STONITH_RC, &(event->result));

	if(safe_str_eq(ntype, T_STONITH_NOTIFY_FENCE)) {
	event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN);
	event->target = crm_element_value_copy(data, F_STONITH_TARGET);
	event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION);
	event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE);
	event->id = crm_element_value_copy(data, F_STONITH_REMOTE);
	}

	g_free(data_addr);
	return event;
	}

	static void
	stonith_send_notification(gpointer data, gpointer user_data)
	{
	struct notify_blob_s *blob = user_data;
	stonith_notify_client_t *entry = data;
	stonith_event_t *st_event = NULL;
	const char *event = NULL;

	if (blob->xml == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(blob->xml, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->notify == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (safe_str_neq(entry->event, event)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	st_event = xml_to_event(blob->xml);

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->notify(blob->stonith, st_event);
	crm_trace("Callback invoked...");
	}

	int
	stonith_send_command(stonith_t * stonith, const char op, xmlNode data, xmlNode ** output_data,
	int call_options, int timeout)
	{
	int rc = 0;
	int reply_id = -1;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	stonith_private_t *native = stonith->private;

	if (stonith->state == stonith_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	stonith->call_id++;
	/* prevent call_id from being negative (or zero) and conflicting
	* with the stonith_errors enum
	* use 2 because we use it as (stonith->call_id - 1) below
	*/
	if (stonith->call_id < 1) {
	stonith->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);
	op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout);
	crm_trace("Sending %s message to STONITH service, Timeout: %ds", op, timeout);

	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, 1000*(timeout + 60));
	free_xml(op_msg);

	if(rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & st_opt_sync_call)) {
	crm_trace("Async call %d, returning", stonith->call_id);
	CRM_CHECK(stonith->call_id != 0, return -EPROTO);
	free_xml(op_reply);

	return stonith->call_id;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id);

	if (reply_id == stonith->call_id) {
	crm_trace("Syncronous reply %d received", reply_id);

	if (crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) {
	rc = -ENOMSG;
	}

	if ((call_options & st_opt_discard_reply) \|\| output_data == NULL) {
	crm_trace("Discarding reply");

	} else {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	free_xml(op_reply);
	rc = -ENOMSG;

	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, stonith->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	free_xml(op_reply);
	rc = -ENOMSG;
	}

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("STONITH disconnected");
	stonith->state = stonith_disconnected;
	}

	free_xml(op_reply);
	return rc;
	}

	/* Not used with mainloop */
	bool
	stonith_dispatch(stonith_t * st)
	{
	gboolean stay_connected = TRUE;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->private;

	while(crm_ipc_ready(private->ipc)) {

	if(crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);
	stonith_dispatch_internal(msg, strlen(msg), st);
	}

	if(crm_ipc_connected(private->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	int
	stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	struct notify_blob_s blob;

	stonith_t * st = userdata;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->private;

	blob.stonith = st;
	blob.xml = string2xml(buffer);
	if (blob.xml == NULL) {
	crm_warn("Received a NULL msg from STONITH service: %s.", buffer);
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(blob.xml, F_TYPE);
	crm_trace("Activating %s callbacks...", type);

	if (safe_str_eq(type, T_STONITH_NG)) {
	stonith_perform_callback(st, blob.xml, 0, 0);

	} else if (safe_str_eq(type, T_STONITH_NOTIFY)) {
	g_list_foreach(private->notify_list, stonith_send_notification, &blob);

	} else {
	crm_err("Unknown message type: %s", type);
	crm_log_xml_warn(blob.xml, "BadReply");
	}

	free_xml(blob.xml);
	return 1;
	}

	static int
	stonith_api_free(stonith_t * stonith)
	{
	int rc = pcmk_ok;

	if (stonith->state != stonith_disconnected) {
	rc = stonith->cmds->disconnect(stonith);
	}

	if (stonith->state == stonith_disconnected) {
	stonith_private_t *private = stonith->private;

	g_hash_table_destroy(private->stonith_op_callback_table);
	free(private->token);
	free(stonith->private);
	free(stonith->cmds);
	free(stonith);
	}

	return rc;
	}

	void
	stonith_api_delete(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->private;
	GList *list = private->notify_list;

	while (list != NULL) {
	stonith_notify_client_t *client = g_list_nth_data(list, 0);

	list = g_list_remove(list, client);
	free(client);
	}

	stonith->cmds->free(stonith);
	stonith = NULL;
	}

	stonith_t *
	stonith_api_new(void)
	{
	stonith_t *new_stonith = NULL;
	stonith_private_t *private = NULL;

	new_stonith = calloc(1, sizeof(stonith_t));
	private = calloc(1, sizeof(stonith_private_t));
	new_stonith->private = private;

	private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal,
	NULL, stonith_destroy_op_callback);
	private->notify_list = NULL;

	new_stonith->call_id = 1;
	new_stonith->state = stonith_disconnected;

	new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));

	/* INDENT-OFF */
	new_stonith->cmds->free = stonith_api_free;
	new_stonith->cmds->connect = stonith_api_signon;
	new_stonith->cmds->disconnect = stonith_api_signoff;

	new_stonith->cmds->list = stonith_api_list;
	new_stonith->cmds->monitor = stonith_api_monitor;
	new_stonith->cmds->status = stonith_api_status;
	new_stonith->cmds->fence = stonith_api_fence;
	new_stonith->cmds->confirm = stonith_api_confirm;
	new_stonith->cmds->history = stonith_api_history;

	new_stonith->cmds->list_agents = stonith_api_device_list;
	new_stonith->cmds->metadata = stonith_api_device_metadata;

	new_stonith->cmds->query = stonith_api_query;
	new_stonith->cmds->remove_device = stonith_api_remove_device;
	new_stonith->cmds->register_device = stonith_api_register_device;

	new_stonith->cmds->remove_level = stonith_api_remove_level;
	new_stonith->cmds->register_level = stonith_api_register_level;

	new_stonith->cmds->remove_callback = stonith_api_del_callback;
	new_stonith->cmds->register_callback = stonith_api_add_callback;
	new_stonith->cmds->remove_notification = stonith_api_del_notification;
	new_stonith->cmds->register_notification = stonith_api_add_notification;
	/* INDENT-ON */

	return new_stonith;
	}

	stonith_key_value_t *
	stonith_key_value_add(stonith_key_value_t * head, const char key, const char value)
	{
	stonith_key_value_t p, end;

	p = calloc(1, sizeof(stonith_key_value_t));
	if (key) {
	p->key = strdup(key);
	}
	if (value) {
	p->value = strdup(value);
	}

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
	{
	stonith_key_value_t *p;

	while (head) {
	p = head->next;
	if (keys) {
	free(head->key);
	}
	if (values) {
	free(head->value);
	}
	free(head);
	head = p;
	}
	}

	int
	stonith_api_kick(int nodeid, const char *uname, int timeout, bool off)
	{
	char *name = NULL;
	const char *action = "reboot";

	int rc = -EPROTO;
	stonith_t *st = NULL;
	enum stonith_call_options opts = st_opt_sync_call \| st_opt_allow_suicide;

	st = stonith_api_new();
	if (st) {
	rc = st->cmds->connect(st, "stonith-api", NULL);
	}

	if (uname != NULL) {
	name = strdup(uname);

	} else if (nodeid > 0) {
	opts \|= st_opt_cs_nodeid;
	name = crm_itoa(nodeid);
	}

	if (off) {
	action = "off";
	}

	if (rc == pcmk_ok) {
	rc = st->cmds->fence(st, opts, name, action, timeout);
	}

	if (st) {
	st->cmds->disconnect(st);
	stonith_api_delete(st);
	}

	free(name);
	return rc;
	}

	time_t
	stonith_api_time(int nodeid, const char *uname, bool in_progress)
	{
	int rc = 0;
	char *name = NULL;

	time_t when = 0;
	time_t progress = 0;
	stonith_t *st = NULL;
	stonith_history_t history, hp = NULL;
	enum stonith_call_options opts = st_opt_sync_call;

	st = stonith_api_new();
	if (st) {
	rc = st->cmds->connect(st, "stonith-api", NULL);
	}

	if (uname != NULL) {
	name = strdup(uname);

	} else if (nodeid > 0) {
	opts \|= st_opt_cs_nodeid;
	name = crm_itoa(nodeid);
	}

	if (st && rc == pcmk_ok) {
	st->cmds->history(st, st_opt_sync_call \| st_opt_cs_nodeid, name, &history, 120);

	for (hp = history; hp; hp = hp->next) {
	if (in_progress) {
	if (hp->state != st_done && hp->state != st_failed) {
	progress = time(NULL);
	}

	} else if (hp->state == st_done) {
	when = hp->completed;
	}
	}
	}

	if (progress) {
	when = progress;
	}

	if (st) {
	st->cmds->disconnect(st);
	stonith_api_delete(st);
	}

	free(name);
	return when;
	}
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index c96ab99979..a7ab7ee198 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,1007 +1,1007 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/msg_xml.h>

	#include <crm/stonith-ng.h>

	CRM_TRACE_INIT_DATA(lrmd);

	static stonith_t *stonith_api = NULL;

	typedef struct lrmd_private_s {
	int call_id;

	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;

	lrmd_event_callback callback;

	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	static void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	/* This will get all the int values.
	* we just have to be careful not to leave any
	* dangling pointers to strings. */
	memcpy(copy, event, sizeof(lrmd_event_data_t));

	copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
	copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
	copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
	copy->output = event->output ? strdup(event->output) : NULL;

	if (event->params) {
	copy->params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (copy->params != NULL) {
	g_hash_table_foreach(event->params, dup_attr, copy->params);
	}
	}

	return copy;
	}

	void
	lrmd_free_event(lrmd_event_data_t * event)
	{
	if (!event) {
	return;
	}

	/* free gives me grief if i try to cast */
	free((char *)event->rsc_id);
	free((char *)event->op_type);
	free((char *)event->user_data);
	free((char *)event->output);
	if (event->params) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static int
	lrmd_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type;
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	lrmd_event_data_t event = { 0, };
	xmlNode *msg;

	if (!native->callback) {
	/* no callback set */
	return 1;
	}

	msg = string2xml(buffer);
	type = crm_element_value(msg, F_LRMD_OPERATION);
	crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);

	if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
	event.type = lrmd_event_register;
	} else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
	event.type = lrmd_event_unregister;
	} else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
	crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
	crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
	crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
	crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
	crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);

	crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
	crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
	crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
	crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);

	event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
	event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
	event.type = lrmd_event_exec_complete;

	event.params = xml2list(msg);
	}

	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	free_xml(msg);
	return 1;
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	gboolean stay_connected = TRUE;
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);
	private = lrmd->private;
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	lrmd_dispatch_internal(msg, strlen(msg), lrmd);
	}
	if (crm_ipc_connected(private->ipc) == FALSE) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static xmlNode *
	lrmd_create_op(int call_id,
	const char token, const char op, xmlNode * data, enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");

	crm_xml_add(op_msg, F_TYPE, T_LRMD);
	crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_LRMD_OPERATION, op);
	crm_xml_add_int(op_msg, F_LRMD_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)options, options);
	crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);

	if (data != NULL) {
	add_message_xml(op_msg, F_LRMD_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	lrmd_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("connection destroyed");
	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	}

	static int
	lrmd_send_command(lrmd_t * lrmd, const char op, xmlNode data, xmlNode ** output_data, int timeout, /* ms. defaults to 1000 if set to 0 */
	enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	int reply_id = -1;
	lrmd_private_t *native = lrmd->private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	if (!native->ipc) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	native->call_id++;
	if (native->call_id < 1) {
	native->call_id = 1;
	}

	CRM_CHECK(native->token != NULL,;);

	op_msg = lrmd_create_op(native->call_id, native->token, op, data, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);

	rc = crm_ipc_send(native->ipc, op_msg, &op_reply, timeout);
	free_xml(op_msg);

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id);
	if (reply_id == native->call_id) {
	crm_trace("reply received");
	if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Recieved bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	rc = -ENOMSG;
	} else {
	crm_err("Recieved bad reply: %d (wanted %d)", reply_id, native->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	rc = -ENOMSG;
	}

	crm_log_xml_trace(op_reply, "Reply");

	done:
	if (crm_ipc_connected(native->ipc) == FALSE) {
	crm_err("LRMD disconnected");
	}

	free_xml(op_reply);
	return rc;
	}

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;

	static struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_dispatch_internal,
	.destroy = lrmd_connection_destroy
	};

	crm_info("Connecting to lrmd");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("lrmd", 0);
	if (native->ipc && crm_ipc_connect(native->ipc)) {
	*fd = crm_ipc_get_fd(native->ipc);
	} else if (native->ipc) {
	rc = -ENOTCONN;
	}
	} else {
	- native->source = mainloop_add_ipc_client("lrmd", 0, lrmd, &lrmd_callbacks);
	+ native->source = mainloop_add_ipc_client("lrmd", G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the LRMD API");
	rc = -ENOTCONN;
	}

	if (!rc) {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "lrmd_command");

	crm_xml_add(hello, F_TYPE, T_LRMD);
	crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_LRMD_CLIENTNAME, name);

	rc = crm_ipc_send(native->ipc, hello, &reply, -1);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);
	}

	return rc;
	}

	static int
	lrmd_api_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	crm_info("Disconnecting from lrmd service");

	if (native->source) {
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;
	} else if (native->ipc) {
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->source = NULL;
	native->ipc = NULL;
	}

	free(native->token);
	native->token = NULL;
	return 0;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (safe_str_eq(class, "ocf") && !provider) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_CLASS, class);
	crm_xml_add(data, F_LRMD_PROVIDER, provider);
	crm_xml_add(data, F_LRMD_TYPE, type);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	lrmd_rsc_info_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_rsc_info_t));

	copy->id = strdup(rsc_info->id);
	copy->type = strdup(rsc_info->type);
	copy->class = strdup(rsc_info->class);
	if (rsc_info->provider) {
	copy->provider = strdup(rsc_info->provider);
	}

	return copy;
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->class);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options);
	free_xml(data);

	class = crm_element_value(output, F_LRMD_CLASS);
	provider = crm_element_value(output, F_LRMD_PROVIDER);
	type = crm_element_value(output, F_LRMD_TYPE);

	if (!output) {
	return NULL;
	} else if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (safe_str_eq(class, "ocf") && !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
	rsc_info->id = strdup(rsc_id);
	rsc_info->class = strdup(class);
	if (provider) {
	rsc_info->provider = strdup(provider);
	}
	rsc_info->type = strdup(type);

	free_xml(output);
	return rsc_info;
	}

	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->private;

	native->callback = callback;
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;

	stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
	if (*output == NULL) {
	rc = -EIO;
	}
	return rc;
	}

	static int
	lsb_get_metadata(const char type, char *output)
	{

	#define lsb_metadata_template \
	"<?xml version=\"1.0\"?>\n"\
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"\
	"<resource-agent name=\"%s\" version=\"0.1\">\n"\
	" <version>1.0</version>\n"\
	" <longdesc lang=\"en\">\n"\
	" %s"\
	" </longdesc>\n"\
	" <shortdesc lang=\"en\">%s</shortdesc>\n"\
	" <parameters>\n"\
	" </parameters>\n"\
	" <actions>\n"\
	" <action name=\"start\" timeout=\"15\" />\n"\
	" <action name=\"stop\" timeout=\"15\" />\n"\
	" <action name=\"status\" timeout=\"15\" />\n"\
	" <action name=\"restart\" timeout=\"15\" />\n"\
	" <action name=\"force-reload\" timeout=\"15\" />\n"\
	" <action name=\"monitor\" timeout=\"15\" interval=\"15\" />\n"\
	" <action name=\"meta-data\" timeout=\"5\" />\n"\
	" </actions>\n"\
	" <special tag=\"LSB\">\n"\
	" <Provides>%s</Provides>\n"\
	" <Required-Start>%s</Required-Start>\n"\
	" <Required-Stop>%s</Required-Stop>\n"\
	" <Should-Start>%s</Should-Start>\n"\
	" <Should-Stop>%s</Should-Stop>\n"\
	" <Default-Start>%s</Default-Start>\n"\
	" <Default-Stop>%s</Default-Stop>\n"\
	" </special>\n"\
	"</resource-agent>\n"

	#define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
	#define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
	#define PROVIDES "# Provides:"
	#define REQ_START "# Required-Start:"
	#define REQ_STOP "# Required-Stop:"
	#define SHLD_START "# Should-Start:"
	#define SHLD_STOP "# Should-Stop:"
	#define DFLT_START "# Default-Start:"
	#define DFLT_STOP "# Default-Stop:"
	#define SHORT_DSCR "# Short-Description:"
	#define DESCRIPTION "# Description:"

	#define lsb_meta_helper_free_value(m) \
	if ((m) != NULL) { \
	xmlFree(m); \
	(m) = NULL; \
	}

	#define lsb_meta_helper_get_value(buffer, ptr, keyword) \
	if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) { \
	(ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \
	continue; \
	}

	char ra_pathname[PATH_MAX] = { 0, };
	FILE *fp;
	GString *meta_data = NULL;
	char buffer[1024];
	char *provides = NULL;
	char *req_start = NULL;
	char *req_stop = NULL;
	char *shld_start = NULL;
	char *shld_stop = NULL;
	char *dflt_start = NULL;
	char *dflt_stop = NULL;
	char *s_dscrpt = NULL;
	char *xml_l_dscrpt = NULL;
	GString *l_dscrpt = NULL;

	snprintf(ra_pathname, sizeof(ra_pathname), "%s%s%s",
	type[0] == '/' ? "" : LSB_ROOT_DIR, type[0] == '/' ? "" : "/", type);

	if (!(fp = fopen(ra_pathname, "r"))) {
	return -EIO;
	}

	/* Enter into the lsb-compliant comment block */
	while (fgets(buffer, sizeof(buffer), fp)) {
	/* Now suppose each of the following eight arguments contain only one line */
	lsb_meta_helper_get_value(buffer, provides, PROVIDES)
	lsb_meta_helper_get_value(buffer, req_start, REQ_START)
	lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP)
	lsb_meta_helper_get_value(buffer, shld_start, SHLD_START)
	lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP)
	lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START)
	lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP)
	lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR)

	/* Long description may cross multiple lines */
	if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) {
	l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION));
	/* Between # and keyword, more than one space, or a tab character,
	* indicates the continuation line. Extracted from LSB init script standard */
	while (fgets(buffer, sizeof(buffer), fp)) {
	if (!strncmp(buffer, "# ", 3) \|\| !strncmp(buffer, "#\t", 2)) {
	buffer[0] = ' ';
	l_dscrpt = g_string_append(l_dscrpt, buffer);
	} else {
	fputs(buffer, fp);
	break; /* Long description ends */
	}
	}
	continue;
	}
	if (l_dscrpt) {
	xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str));
	}
	if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) {
	/* Get to the out border of LSB comment block */
	break;
	}
	if (buffer[0] != '#') {
	break; /* Out of comment block in the beginning */
	}
	}
	fclose(fp);

	meta_data = g_string_new("");
	g_string_sprintf(meta_data, lsb_metadata_template, type,
	(xml_l_dscrpt == NULL) ? type : xml_l_dscrpt,
	(s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides,
	(req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop,
	(shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop,
	(dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop);

	lsb_meta_helper_free_value(xml_l_dscrpt);
	lsb_meta_helper_free_value(s_dscrpt);
	lsb_meta_helper_free_value(provides);
	lsb_meta_helper_free_value(req_start);
	lsb_meta_helper_free_value(req_stop);
	lsb_meta_helper_free_value(shld_start);
	lsb_meta_helper_free_value(shld_stop);
	lsb_meta_helper_free_value(dflt_start);
	lsb_meta_helper_free_value(dflt_stop);

	if (l_dscrpt) {
	g_string_free(l_dscrpt, TRUE);
	}

	*output = strdup(meta_data->str);
	g_string_free(meta_data, TRUE);

	return pcmk_ok;
	}

	static int
	generic_get_metadata(const char standard, const char provider, const char type, char *output)
	{
	svc_action_t *action = resources_action_create(type,
	standard,
	provider,
	type,
	"meta-data",
	0,
	5000,
	NULL);

	if (!(services_action_sync(action))) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_get_metadata(lrmd_t * lrmd,
	const char *class,
	const char *provider,
	const char type, char *output, enum lrmd_call_options options)
	{
	if (!class \|\| !type) {
	return -EINVAL;
	}

	if (safe_str_eq(class, "stonith")) {
	return stonith_get_metadata(provider, type, output);
	} else if (safe_str_eq(class, "lsb")) {
	return lsb_get_metadata(type, output);
	}
	return generic_get_metadata(class, provider, type, output);
	}

	static int
	lrmd_api_exec(lrmd_t * lrmd, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t * lrmd, const char rsc_id, const char action, int interval)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if(resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;

	if (safe_str_eq(class, "stonith")) {
	rc += list_stonith_agents(resources);

	} else {
	GListPtr gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	rc += list_stonith_agents(resources);
	}
	}

	if(rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static int
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	int found = 0;
	GList *agents = NULL;
	GListPtr gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (safe_str_eq(agent, gIter2->data)) {
	found = 1;
	}
	}
	g_list_free_full(agents, free);

	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GListPtr gIter = NULL;

	ocf_providers = resources_list_providers("ocf");

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider, "ocf")) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	char *standard = NULL;
	GList *standards = NULL;
	GListPtr gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	standard = gIter->data;
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if(list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, "stonith");
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *new_lrmd = NULL;
	lrmd_private_t *pvt = NULL;

	new_lrmd = calloc(1, sizeof(lrmd_t));
	pvt = calloc(1, sizeof(lrmd_private_t));
	new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	new_lrmd->private = pvt;

	new_lrmd->cmds->connect = lrmd_api_connect;
	new_lrmd->cmds->disconnect = lrmd_api_disconnect;
	new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
	new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	new_lrmd->cmds->set_callback = lrmd_api_set_callback;
	new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
	new_lrmd->cmds->exec = lrmd_api_exec;
	new_lrmd->cmds->cancel = lrmd_api_cancel;
	new_lrmd->cmds->list_agents = lrmd_api_list_agents;
	new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	new_lrmd->cmds->list_standards = lrmd_api_list_standards;

	if (!stonith_api) {
	stonith_api = stonith_api_new();
	}

	return new_lrmd;
	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	lrmd->cmds->disconnect(lrmd); /* no-op if already disconnected */
	free(lrmd->cmds);
	free(lrmd->private);
	free(lrmd);
	}
	diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
	index 7057e8df49..af3172ad56 100644
	--- a/lib/services/services_linux.c
	+++ b/lib/services/services_linux.c
	@@ -1,554 +1,556 @@
	/*
	* Copyright (C) 2010 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	#define _GNU_SOURCE
	#endif

	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <unistd.h>
	#include <dirent.h>
	#include <fcntl.h>
	#include <string.h>

	#include "crm/crm.h"
	#include "crm/common/mainloop.h"
	#include "crm/services.h"

	#include "services_private.h"

	static inline void
	set_fd_opts(int fd, int opts)
	{
	int flag;
	if ((flag = fcntl(fd, F_GETFL)) >= 0) {
	if (fcntl(fd, F_SETFL, flag \| opts) < 0) {
	crm_err( "fcntl() write failed");
	}
	} else {
	crm_err( "fcntl() read failed");
	}
	}

	static gboolean
	read_output(int fd, svc_action_t *op)
	{
	char *data = NULL;
	int rc = 0, len = 0;
	gboolean is_err = FALSE;
	char buf[500];
	static const size_t buf_read_len = sizeof(buf) - 1;

	crm_trace("%p", op);

	if (fd < 0) {
	return FALSE;
	}

	if (fd == op->opaque->stderr_fd) {
	is_err = TRUE;
	if (op->stderr_data) {
	len = strlen(op->stderr_data);
	data = op->stderr_data;
	}
	} else if (op->stdout_data) {
	len = strlen(op->stdout_data);
	data = op->stdout_data;
	}

	do {
	rc = read(fd, buf, buf_read_len);
	if (rc > 0) {
	buf[rc] = 0;
	data = realloc(data, len + rc + 1);
	sprintf(data + len, "%s", buf);
	len += rc;
	} else if (errno != EINTR) {
	/* error or EOF
	* Cleanup happens in pipe_done()
	*/
	rc = FALSE;
	break;
	}

	} while (rc == buf_read_len \|\| rc < 0);

	if (data != NULL && is_err) {
	op->stderr_data = data;
	} else if (data != NULL) {
	op->stdout_data = data;
	}

	return rc;
	}

	static int
	dispatch_stdout(gpointer userdata)
	{
	svc_action_t* op = (svc_action_t *) userdata;
	return read_output(op->opaque->stdout_fd, op);
	}

	static int
	dispatch_stderr(gpointer userdata)
	{
	svc_action_t* op = (svc_action_t *) userdata;
	return read_output(op->opaque->stderr_fd, op);
	}

	static void
	pipe_out_done(gpointer user_data)
	{
	svc_action_t* op = (svc_action_t *) user_data;

	crm_trace("%p", op);

	op->opaque->stdout_gsource = NULL;
	if (op->opaque->stdout_fd > STDOUT_FILENO) {
	close(op->opaque->stdout_fd);
	}
	op->opaque->stdout_fd = -1;
	}

	static void
	pipe_err_done(gpointer user_data)
	{
	svc_action_t* op = (svc_action_t *) user_data;
	op->opaque->stderr_gsource = NULL;
	if (op->opaque->stderr_fd > STDERR_FILENO) {
	close(op->opaque->stderr_fd);
	}
	op->opaque->stderr_fd = -1;
	}

	static struct mainloop_fd_callbacks stdout_callbacks = {
	.dispatch = dispatch_stdout,
	.destroy = pipe_out_done,
	};

	static struct mainloop_fd_callbacks stderr_callbacks = {
	.dispatch = dispatch_stderr,
	.destroy = pipe_err_done,
	};

	static void
	set_ocf_env(const char key, const char value, gpointer user_data)
	{
	if (setenv(key, value, 1) != 0) {
	crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
	}
	}

	static void
	set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
	{
	char buffer[500];
	snprintf(buffer, sizeof(buffer), "OCF_RESKEY_%s", (char *) key);
	set_ocf_env(buffer, value, user_data);
	}

	static void
	add_OCF_env_vars(svc_action_t *op)
	{
	if (!op->standard \|\| strcasecmp("ocf", op->standard) != 0) {
	return;
	}

	if (op->params) {
	g_hash_table_foreach(op->params, set_ocf_env_with_prefix, NULL);
	}

	set_ocf_env("OCF_RA_VERSION_MAJOR", "1", NULL);
	set_ocf_env("OCF_RA_VERSION_MINOR", "0", NULL);
	set_ocf_env("OCF_ROOT", OCF_ROOT_DIR, NULL);

	if (op->rsc) {
	set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
	}

	if (op->agent != NULL) {
	set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
	}

	/* Notes: this is not added to specification yet. Sept 10,2004 */
	if (op->provider != NULL) {
	set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
	}
	}

	gboolean recurring_action_timer(gpointer data)
	{
	svc_action_t *op = data;
	crm_debug("Scheduling another invokation of %s", op->id);

	/* Clean out the old result */
	free(op->stdout_data); op->stdout_data = NULL;
	free(op->stderr_data); op->stderr_data = NULL;

	services_action_async(op, NULL);
	return FALSE;
	}

	void
	operation_finalize(svc_action_t *op)
	{
	int recurring = 0;
	if (op->interval) {
	if (op->cancel) {
	op->status = PCMK_LRM_OP_CANCELLED;
	cancel_recurring_action(op);
	} else {
	recurring = 1;
	op->opaque->repeat_timer = g_timeout_add(op->interval,
	recurring_action_timer,
	(void *) op);
	}
	}

	if (op->opaque->callback) {
	op->opaque->callback(op);
	}

	if (!recurring) {
	/*
	* If this is a recurring action, do not free explicitly.
	* It will get freed whenever the action gets cancelled.
	*/
	services_action_free(op);
	}
	}

	static void
	operation_finished(mainloop_child_t *p, int status, int signo, int exitcode)
	{
	char *next = NULL;
	char *offset = NULL;
	svc_action_t *op = mainloop_get_child_userdata(p);
	pid_t pid = mainloop_get_child_pid(p);

	mainloop_clear_child_userdata(p);
	op->status = PCMK_LRM_OP_DONE;
	CRM_ASSERT(op->pid == pid);

	if (op->opaque->stderr_gsource) {
	/* Make sure we have read everything from the buffer.
	* Depending on the priority mainloop gives the fd, operation_finished
	* could occur before all the reads are done. Force the read now.*/
	dispatch_stderr(op);
	}

	if (op->opaque->stdout_gsource) {
	/* Make sure we have read everything from the buffer.
	* Depending on the priority mainloop gives the fd, operation_finished
	* could occur before all the reads are done. Force the read now.*/
	dispatch_stdout(op);
	}

	if (signo) {
	if (mainloop_get_child_timeout(p)) {
	crm_warn("%s:%d - timed out after %dms", op->id, op->pid,
	op->timeout);
	op->status = PCMK_LRM_OP_TIMEOUT;
	op->rc = PCMK_OCF_TIMEOUT;

	} else {
	crm_warn("%s:%d - terminated with signal %d", op->id, op->pid,
	signo);
	op->status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_OCF_SIGNAL;
	}

	} else {
	op->rc = exitcode;
	crm_debug("%s:%d - exited with rc=%d", op->id, op->pid, exitcode);

	if (op->stdout_data) {
	next = op->stdout_data;
	do {
	offset = next;
	next = strchrnul(offset, '\n');
	crm_debug("%s:%d [ %.*s ]", op->id, op->pid,
	(int) (next - offset), offset);
	if (next[0] != 0) {
	next++;
	}

	} while (next != NULL && next[0] != 0);
	}

	if (op->stderr_data) {
	next = op->stderr_data;
	do {
	offset = next;
	next = strchrnul(offset, '\n');
	crm_notice("%s:%d [ %.*s ]", op->id, op->pid,
	(int) (next - offset), offset);
	if (next[0] != 0) {
	next++;
	}

	} while (next != NULL && next[0] != 0);
	}
	}

	op->pid = 0;
	operation_finalize(op);
	}

	gboolean
	services_os_action_execute(svc_action_t* op, gboolean synchronous)
	{
	int rc, lpc;
	int stdout_fd[2];
	int stderr_fd[2];

	if (pipe(stdout_fd) < 0) {
	crm_err( "pipe() failed");
	}

	if (pipe(stderr_fd) < 0) {
	crm_err( "pipe() failed");
	}

	op->pid = fork();
	switch (op->pid) {
	case -1:
	crm_err( "fork() failed");
	close(stdout_fd[0]);
	close(stdout_fd[1]);
	close(stderr_fd[0]);
	close(stderr_fd[1]);
	return FALSE;

	case 0: /* Child */
	/* Man: The call setpgrp() is equivalent to setpgid(0,0)
	* _and_ compiles on BSD variants too
	* need to investigate if it works the same too.
	*/
	setpgid(0, 0);
	close(stdout_fd[0]);
	close(stderr_fd[0]);
	if (STDOUT_FILENO != stdout_fd[1]) {
	if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
	crm_err( "dup2() failed (stdout)");
	}
	close(stdout_fd[1]);
	}
	if (STDERR_FILENO != stderr_fd[1]) {
	if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
	crm_err( "dup2() failed (stderr)");
	}
	close(stderr_fd[1]);
	}

	/* close all descriptors except stdin/out/err and channels to logd */
	for (lpc = getdtablesize() - 1; lpc > STDERR_FILENO; lpc--) {
	close(lpc);
	}

	/* Setup environment correctly */
	add_OCF_env_vars(op);

	/* execute the RA */
	execvp(op->opaque->exec, op->opaque->args);

	switch (errno) { /* see execve(2) */
	case ENOENT: /* No such file or directory */
	case EISDIR: /* Is a directory */
	rc = PCMK_OCF_NOT_INSTALLED;
	break;
	case EACCES: /* permission denied (various errors) */
	rc = PCMK_OCF_INSUFFICIENT_PRIV;
	break;
	default:
	rc = PCMK_OCF_UNKNOWN_ERROR;
	break;
	}
	_exit(rc);
	}

	/* Only the parent reaches here */
	close(stdout_fd[1]);
	close(stderr_fd[1]);

	op->opaque->stdout_fd = stdout_fd[0];
	set_fd_opts(op->opaque->stdout_fd, O_NONBLOCK);

	op->opaque->stderr_fd = stderr_fd[0];
	set_fd_opts(op->opaque->stderr_fd, O_NONBLOCK);

	if (synchronous) {
	int status = 0;
	int timeout = (1 + op->timeout) / 1000;
	crm_trace("Waiting for %d", op->pid);
	while ((op->timeout < 0 \|\| timeout > 0) && waitpid(op->pid, &status, WNOHANG) <= 0) {
	sleep(1);
	read_output(op->opaque->stdout_fd, op);
	read_output(op->opaque->stderr_fd, op);
	timeout--;
	}

	crm_trace("Child done: %d", op->pid);
	if (timeout == 0) {
	int killrc = kill(op->pid, 9 /SIGKILL/);

	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->status = PCMK_LRM_OP_TIMEOUT;
	crm_warn("%s:%d - timed out after %dms", op->id, op->pid,
	op->timeout);

	if (killrc && errno != ESRCH) {
	crm_err("kill(%d, KILL) failed: %d", op->pid, errno);
	}

	} else if (WIFEXITED(status)) {
	op->status = PCMK_LRM_OP_DONE;
	op->rc = WEXITSTATUS(status);
	crm_info("Managed %s process %d exited with rc=%d", op->id, op->pid,
	op->rc);

	} else if (WIFSIGNALED(status)) {
	int signo = WTERMSIG(status);
	op->status = PCMK_LRM_OP_ERROR;
	crm_err("Managed %s process %d exited with signal=%d", op->id,
	op->pid, signo);
	}
	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	crm_err("Managed %s process %d dumped core", op->id, op->pid);
	}
	#endif

	read_output(op->opaque->stdout_fd, op);
	read_output(op->opaque->stderr_fd, op);

	} else {
	crm_trace("Async waiting for %d - %s", op->pid, op->opaque->exec);
	mainloop_add_child(op->pid, op->timeout, op->id, op,
	operation_finished);

	op->opaque->stdout_gsource = mainloop_add_fd(op->id,
	+ G_PRIORITY_LOW,
	op->opaque->stdout_fd,
	op,
	&stdout_callbacks);

	op->opaque->stderr_gsource = mainloop_add_fd(op->id,
	+ G_PRIORITY_LOW,
	op->opaque->stderr_fd,
	op,
	&stderr_callbacks);
	}

	return TRUE;
	}

	GList *
	services_os_get_directory_list(const char *root, gboolean files)
	{
	GList *list = NULL;
	struct dirent **namelist;
	int entries = 0, lpc = 0;
	char buffer[PATH_MAX];

	entries = scandir(root, &namelist, NULL, alphasort);
	if (entries <= 0) {
	return list;
	}

	for (lpc = 0; lpc < entries; lpc++) {
	struct stat sb;
	if ('.' == namelist[lpc]->d_name[0]) {
	free(namelist[lpc]);
	continue;
	}

	snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);

	if (stat(buffer, &sb)) {
	continue;
	}

	if (S_ISDIR(sb.st_mode)) {
	if (files) {
	free(namelist[lpc]);
	continue;
	}

	} else if (S_ISREG(sb.st_mode)) {
	if (files == FALSE) {
	free(namelist[lpc]);
	continue;

	} else if ((sb.st_mode & S_IXUSR) == 0
	&& (sb.st_mode & S_IXGRP) == 0
	&& (sb.st_mode & S_IXOTH) == 0) {
	free(namelist[lpc]);
	continue;
	}
	}

	list = g_list_append(list, strdup(namelist[lpc]->d_name));

	free(namelist[lpc]);
	}

	free(namelist);
	return list;
	}

	GList *
	resources_os_list_lsb_agents(void)
	{
	return get_directory_list(LSB_ROOT_DIR, TRUE);
	}

	GList *
	resources_os_list_ocf_providers(void)
	{
	return get_directory_list(OCF_ROOT_DIR "/resource.d", FALSE);
	}

	GList *
	resources_os_list_ocf_agents(const char *provider)
	{
	GList *gIter = NULL;
	GList *result = NULL;
	GList *providers = NULL;

	if (provider) {
	char buffer[500];
	snprintf(buffer, sizeof(buffer), "%s/resource.d/%s", OCF_ROOT_DIR,
	provider);
	return get_directory_list(buffer, TRUE);
	}

	providers = resources_os_list_ocf_providers();
	for (gIter = providers; gIter != NULL; gIter = gIter->next) {
	GList *tmp1 = result;
	GList *tmp2 = resources_os_list_ocf_agents(gIter->data);
	if(tmp2) {
	result = g_list_concat(tmp1, tmp2);
	}
	}
	g_list_free_full(providers, free);
	return result;
	}
	diff --git a/mcp/corosync.c b/mcp/corosync.c
	index 57e7c5840a..0f4aa072da 100644
	--- a/mcp/corosync.c
	+++ b/mcp/corosync.c
	@@ -1,649 +1,649 @@
	/*
	* Copyright (C) 2010 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>
	#include <pacemaker.h>

	#include <sys/utsname.h>
	#include <sys/stat.h> /* for calls to stat() */
	#include <libgen.h> /* For basename() and dirname() */

	#include <sys/types.h>
	#include <pwd.h> /* For getpwname() */

	#include <corosync/hdb.h>
	#include <corosync/cfg.h>
	#include <corosync/cpg.h>
	#if HAVE_CONFDB
	# include <corosync/confdb.h>
	#endif

	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#if SUPPORT_CMAN
	# include <libcman.h>
	#endif

	#if HAVE_CMAP
	# include <corosync/cmap.h>
	#endif

	static struct cpg_name cpg_group = {
	.length = 0,
	.value[0] = 0,
	};

	gboolean use_cman = FALSE;
	static cpg_handle_t cpg_handle;
	static corosync_cfg_handle_t cfg_handle;

	/* =::=::=::= CFG - Shutdown stuff =::=::=::= */

	static void
	cfg_shutdown_callback(corosync_cfg_handle_t h, corosync_cfg_shutdown_flags_t flags)
	{
	crm_info("Corosync wants to shut down: %s",
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_IMMEDIATE) ? "immediate" :
	(flags == COROSYNC_CFG_SHUTDOWN_FLAG_REGARDLESS) ? "forced" : "optional");

	/* Never allow corosync to shut down while we're running */
	corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO);
	}

	static corosync_cfg_callbacks_t cfg_callbacks = {
	.corosync_cfg_shutdown_callback = cfg_shutdown_callback,
	};

	static int
	pcmk_cfg_dispatch(gpointer user_data)
	{
	corosync_cfg_handle_t handle = (corosync_cfg_handle_t ) user_data;
	cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL);

	if (rc != CS_OK) {
	return -1;
	}
	return 0;
	}

	static void
	cfg_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	cfg_handle = 0;

	pcmk_shutdown(SIGTERM);
	}

	gboolean
	cluster_disconnect_cfg(void)
	{
	if (cfg_handle) {
	corosync_cfg_finalize(cfg_handle);
	cfg_handle = 0;
	}

	pcmk_shutdown(SIGTERM);
	return TRUE;
	}

	#define cs_repeat(counter, max, code) do { \
	code; \
	if(rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) { \
	counter++; \
	crm_debug("Retrying operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while(counter < max)

	gboolean
	cluster_connect_cfg(uint32_t * nodeid)
	{
	cs_error_t rc;
	int fd = 0, retries = 0;
	static struct mainloop_fd_callbacks cfg_fd_callbacks =
	{
	.dispatch = pcmk_cfg_dispatch,
	.destroy = cfg_connection_destroy,
	};

	cs_repeat(retries, 30, rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks));

	if (rc != CS_OK) {
	crm_err("corosync cfg init error %d", rc);
	return FALSE;
	}

	rc = corosync_cfg_fd_get(cfg_handle, &fd);
	if (rc != CS_OK) {
	crm_err("corosync cfg fd_get error %d", rc);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = corosync_cfg_local_get(cfg_handle, nodeid));

	if (rc != CS_OK) {
	crm_err("corosync cfg local_get error %d", rc);
	goto bail;
	}

	crm_debug("Our nodeid: %d", *nodeid);
	- mainloop_add_fd("corosync-cfg", fd, &cfg_handle, &cfg_fd_callbacks);
	+ mainloop_add_fd("corosync-cfg", G_PRIORITY_DEFAULT, fd, &cfg_handle, &cfg_fd_callbacks);

	return TRUE;

	bail:
	corosync_cfg_finalize(cfg_handle);
	return FALSE;
	}

	/* =::=::=::= CPG - Closed Process Group Messaging =::=::=::= */

	static int
	pcmk_cpg_dispatch(gpointer user_data)
	{
	cpg_handle_t handle = (cpg_handle_t ) user_data;
	cs_error_t rc = cpg_dispatch(*handle, CS_DISPATCH_ALL);

	if (rc != CS_OK) {
	return -1;
	}
	return 0;
	}

	static void
	cpg_connection_destroy(gpointer user_data)
	{
	crm_err("Connection destroyed");
	cpg_handle = 0;
	}

	static void
	pcmk_cpg_deliver(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	if (nodeid != local_nodeid) {
	uint32_t procs = 0;
	xmlNode *xml = string2xml(msg);
	const char *uname = crm_element_value(xml, "uname");

	crm_element_value_int(xml, "proclist", (int *)&procs);
	/* crm_debug("Got proclist %.32x from %s", procs, uname); */
	if (update_node_processes(nodeid, uname, procs)) {
	update_process_clients();
	}
	}
	}

	static void
	pcmk_cpg_membership(cpg_handle_t handle,
	const struct cpg_name *groupName,
	const struct cpg_address *member_list, size_t member_list_entries,
	const struct cpg_address *left_list, size_t left_list_entries,
	const struct cpg_address *joined_list, size_t joined_list_entries)
	{
	/* Don't care about CPG membership */
	update_process_peers();
	}

	cpg_callbacks_t cpg_callbacks = {
	.cpg_deliver_fn = pcmk_cpg_deliver,
	.cpg_confchg_fn = pcmk_cpg_membership,
	};

	gboolean
	cluster_disconnect_cpg(void)
	{
	if (cpg_handle) {
	cpg_finalize(cpg_handle);
	cpg_handle = 0;
	}
	return TRUE;
	}

	gboolean
	cluster_connect_cpg(void)
	{
	cs_error_t rc;
	unsigned int nodeid;
	int fd;
	int retries = 0;
	static struct mainloop_fd_callbacks cpg_fd_callbacks =
	{
	.dispatch = pcmk_cpg_dispatch,
	.destroy = cpg_connection_destroy,
	};

	strcpy(cpg_group.value, "pcmk");
	cpg_group.length = strlen(cpg_group.value) + 1;

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_initialize(&cpg_handle, &cpg_callbacks));
	if (rc != CS_OK) {
	crm_err("corosync cpg init error %d", rc);
	return FALSE;
	}

	rc = cpg_fd_get(cpg_handle, &fd);
	if (rc != CS_OK) {
	crm_err("corosync cpg fd_get error %d", rc);
	goto bail;
	}

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_local_get(cpg_handle, &nodeid));
	if (rc != CS_OK) {
	crm_err("corosync cpg local_get error %d", rc);
	goto bail;
	}

	crm_debug("Our nodeid: %d", nodeid);

	retries = 0;
	cs_repeat(retries, 30, rc = cpg_join(cpg_handle, &cpg_group));

	if (rc != CS_OK) {
	crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
	goto bail;
	}

	- mainloop_add_fd("corosync-cpg", fd, &cpg_handle, &cpg_fd_callbacks);
	+ mainloop_add_fd("corosync-cpg", G_PRIORITY_DEFAULT, fd, &cpg_handle, &cpg_fd_callbacks);
	return TRUE;

	bail:
	cpg_finalize(cpg_handle);
	return FALSE;
	}

	gboolean
	send_cpg_message(struct iovec * iov)
	{
	int rc = CS_OK;
	int retries = 0;

	errno = 0;

	do {
	rc = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 1);
	if (rc == CS_ERR_TRY_AGAIN \|\| rc == CS_ERR_QUEUE_FULL) {
	cpg_flow_control_state_t fc_state = CPG_FLOW_CONTROL_DISABLED;
	int rc2 = cpg_flow_control_state_get(cpg_handle, &fc_state);

	if (rc2 == CS_OK && fc_state == CPG_FLOW_CONTROL_ENABLED) {
	crm_debug("Attempting to clear cpg dispatch queue");
	rc2 = cpg_dispatch(cpg_handle, CS_DISPATCH_ALL);
	}

	if (rc2 != CS_OK) {
	crm_warn("Could not check/clear the cpg connection");
	goto bail;

	} else {
	retries++;
	crm_debug("Retrying operation after %ds", retries);
	sleep(retries);
	}
	} else {
	break;
	}

	/* 5 retires is plenty, we'll resend once the membership reforms anyway */
	} while (retries < 5);

	bail:
	if (rc != CS_OK) {
	crm_err("Sending message via cpg FAILED: (rc=%d) %s", rc, ais_error2text(rc));
	}

	return (rc == CS_OK);
	}

	/* =::=::=::= Configuration =::=::=::= */
	#if HAVE_CONFDB
	static int
	get_config_opt(confdb_handle_t config,
	hdb_handle_t object_handle, const char key, char value, const char fallback)
	{
	size_t len = 0;
	char *env_key = NULL;
	const char *env_value = NULL;
	char buffer[256];

	if (*value) {
	free(*value);
	*value = NULL;
	}

	if (object_handle > 0) {
	if (CS_OK == confdb_key_get(config, object_handle, key, strlen(key), &buffer, &len)) {
	*value = strdup(buffer);
	}
	}

	if (*value) {
	crm_info("Found '%s' for option: %s", *value, key);
	return 0;
	}

	env_key = crm_concat("HA", key, '_');
	env_value = getenv(env_key);
	free(env_key);

	if (*value) {
	crm_info("Found '%s' in ENV for option: %s", *value, key);
	*value = strdup(env_value);
	return 0;
	}

	if (fallback) {
	crm_info("Defaulting to '%s' for option: %s", fallback, key);
	*value = strdup(fallback);

	} else {
	crm_info("No default for option: %s", key);
	}

	return -1;
	}

	static confdb_handle_t
	config_find_init(confdb_handle_t config)
	{
	cs_error_t rc = CS_OK;
	confdb_handle_t local_handle = OBJECT_PARENT_HANDLE;

	rc = confdb_object_find_start(config, local_handle);
	if (rc == CS_OK) {
	return local_handle;
	} else {
	crm_err("Couldn't create search context: %d", rc);
	}
	return 0;
	}

	static hdb_handle_t
	config_find_next(confdb_handle_t config, const char *name, confdb_handle_t top_handle)
	{
	cs_error_t rc = CS_OK;
	hdb_handle_t local_handle = 0;

	if (top_handle == 0) {
	crm_err("Couldn't search for %s: no valid context", name);
	return 0;
	}

	crm_trace("Searching for %s in " HDB_X_FORMAT, name, top_handle);
	rc = confdb_object_find(config, top_handle, name, strlen(name), &local_handle);
	if (rc != CS_OK) {
	crm_info("No additional configuration supplied for: %s", name);
	local_handle = 0;
	} else {
	crm_info("Processing additional %s options...", name);
	}
	return local_handle;
	}
	#else
	static int
	get_config_opt(cmap_handle_t object_handle, const char key, char value, const char fallback)
	{
	int rc = 0, retries = 0;

	cs_repeat(retries, 5, rc = cmap_get_string(object_handle, key, value));
	if(rc != CS_OK) {
	crm_trace("Search for %s failed %d, defaulting to %s", key, rc, fallback);
	if(fallback) {
	*value = strdup(fallback);
	} else {
	*value = NULL;
	}
	}
	crm_trace("%s: %s", key, *value);
	return rc;
	}

	#endif

	char *
	get_local_node_name(void)
	{
	char *name = NULL;
	struct utsname res;

	if (use_cman) {
	#if SUPPORT_CMAN
	cman_node_t us;
	cman_handle_t cman;

	cman = cman_init(NULL);
	if (cman != NULL && cman_is_active(cman)) {
	us.cn_name[0] = 0;
	cman_get_node(cman, CMAN_NODEID_US, &us);
	name = strdup(us.cn_name);
	crm_info("Using CMAN node name: %s", name);

	} else {
	crm_err("Couldn't determin node name from CMAN");
	}

	cman_finish(cman);
	#endif

	} else if (uname(&res) < 0) {
	crm_perror(LOG_ERR, "Could not determin the current host");
	exit(100);

	} else {
	name = strdup(res.nodename);
	}
	return name;
	}


	gboolean
	read_config(void)
	{
	int rc = CS_OK;
	int retries = 0;
	gboolean have_log = FALSE;

	char *logging_debug = NULL;
	char *logging_logfile = NULL;
	char *logging_to_logfile = NULL;
	char *logging_to_syslog = NULL;
	char *logging_syslog_facility = NULL;

	enum cluster_type_e stack = pcmk_cluster_unknown;

	#if HAVE_CONFDB
	char *value = NULL;
	confdb_handle_t config;
	confdb_handle_t top_handle = 0;
	hdb_handle_t local_handle;
	static confdb_callbacks_t callbacks = { };

	do {
	rc = confdb_initialize(&config, &callbacks);
	if(rc != CS_OK) {
	retries++;
	printf("Connection setup failed: %d. Retrying in %ds\n", rc, retries);
	sleep(retries);

	} else {
	break;
	}

	} while(retries < 5);
	#elif HAVE_CMAP
	cmap_handle_t local_handle;

	/* There can be only one (possibility if confdb isn't around) */
	do {
	rc = cmap_initialize(&local_handle);
	if(rc != CS_OK) {
	retries++;
	printf("API connection setup failed: %s. Retrying in %ds\n",
	cs_strerror(rc), retries);
	crm_info("API connection setup failed: %s. Retrying in %ds",
	cs_strerror(rc), retries);
	sleep(retries);

	} else {
	break;
	}

	} while(retries < 5);
	#endif

	if (rc != CS_OK) {
	printf("Could not connect to Cluster Configuration Database API, error %d\n", rc);
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc);
	return FALSE;
	}

	stack = get_cluster_type();
	crm_info("Reading configure for stack: %s", name_for_cluster_type(stack));

	/* =::=::= Should we be here =::=::= */
	if (stack == pcmk_cluster_corosync) {
	setenv("HA_cluster_type", "corosync", 1);
	setenv("HA_quorum_type", "corosync", 1);

	#if HAVE_CONFDB
	} else if (stack == pcmk_cluster_cman) {
	setenv("HA_cluster_type", "cman", 1);
	setenv("HA_quorum_type", "cman", 1);
	enable_crmd_as_root(TRUE);
	use_cman = TRUE;

	} else if (stack == pcmk_cluster_classic_ais) {
	setenv("HA_cluster_type", "openais", 1);
	setenv("HA_quorum_type", "pcmk", 1);

	/* Look for a service block to indicate our plugin is loaded */
	top_handle = config_find_init(config);
	local_handle = config_find_next(config, "service", top_handle);

	while (local_handle) {
	free(value);
	get_config_opt(config, local_handle, "name", &value, NULL);
	if (safe_str_eq("pacemaker", value)) {
	free(value);
	get_config_opt(config, local_handle, "ver", &value, "0");
	if (safe_str_eq(value, "1")) {
	free(value);
	get_config_opt(config, local_handle, "use_logd", &value, "no");
	setenv("HA_use_logd", value, 1);
	setenv("HA_LOGD", value, 1);

	free(value);
	get_config_opt(config, local_handle, "use_mgmtd", &value, "no");
	enable_mgmtd(crm_is_true(value));

	} else {
	crm_err("We can only start Pacemaker from init if using version 1"
	" of the Pacemaker plugin for Corosync. Terminating.");
	exit(100);
	}
	break;
	}
	local_handle = config_find_next(config, "service", top_handle);
	}
	free(value);

	#endif
	} else {
	crm_err("Unsupported stack type: %s", name_for_cluster_type(stack));
	return FALSE;
	}

	#if HAVE_CONFDB
	top_handle = config_find_init(config);
	local_handle = config_find_next(config, "logging", top_handle);

	get_config_opt(config, local_handle, "debug", &logging_debug, "off");
	get_config_opt(config, local_handle, "logfile", &logging_logfile, "/var/log/pacemaker");
	get_config_opt(config, local_handle, "to_logfile", &logging_to_logfile, "off");
	get_config_opt(config, local_handle, "to_syslog", &logging_to_syslog, "on");
	get_config_opt(config, local_handle, "syslog_facility", &logging_syslog_facility, "daemon");

	confdb_finalize(config);
	#elif HAVE_CMAP
	/* =::=::= Logging =::=::= */
	get_config_opt(local_handle, "logging.debug", &logging_debug, "off");
	get_config_opt(local_handle, "logging.logfile", &logging_logfile, "/var/log/pacemaker");
	get_config_opt(local_handle, "logging.to_logfile", &logging_to_logfile, "off");
	get_config_opt(local_handle, "logging.to_syslog", &logging_to_syslog, "on");
	get_config_opt(local_handle, "logging.syslog_facility", &logging_syslog_facility, "daemon");

	cmap_finalize(local_handle);
	#endif

	if (crm_is_true(logging_debug)) {
	setenv("HA_debug", "1", 1);
	if(get_crm_log_level() < LOG_DEBUG) {
	set_crm_log_level(LOG_DEBUG);
	}

	} else {
	setenv("HA_debug", "0", 1);
	}

	if (crm_is_true(logging_to_logfile)) {
	if(crm_add_logfile(logging_logfile)) {
	setenv("HA_debugfile", logging_logfile, 1);
	setenv("HA_DEBUGLOG", logging_logfile, 1);
	setenv("HA_LOGFILE", logging_logfile, 1);
	have_log = TRUE;

	} else {
	crm_err("Couldn't create logfile: %s", logging_logfile);
	}
	}

	if (have_log && crm_is_true(logging_to_syslog) == FALSE) {
	crm_info("User configured file based logging and explicitly disabled syslog.");
	free(logging_syslog_facility);
	logging_syslog_facility = NULL;

	} else {
	if (crm_is_true(logging_to_syslog) == FALSE) {
	crm_err
	("Please enable some sort of logging, either 'to_logfile: on' or 'to_syslog: on'.");
	crm_err("If you use file logging, be sure to also define a value for 'logfile'");
	}
	}

	if(logging_syslog_facility) {
	setenv("HA_logfacility", logging_syslog_facility, 1);
	setenv("HA_LOGFACILITY", logging_syslog_facility, 1);
	} else {
	unsetenv("HA_logfacility");
	unsetenv("HA_LOGFACILITY");
	}

	free(logging_debug);
	free(logging_logfile);
	free(logging_to_logfile);
	free(logging_to_syslog);
	free(logging_syslog_facility);
	return TRUE;
	}
	diff --git a/tools/crm_node.c b/tools/crm_node.c
	index 6540a0f24d..1aa6a18749 100644
	--- a/tools/crm_node.c
	+++ b/tools/crm_node.c
	@@ -1,779 +1,779 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <libgen.h> /* for basename() */

	#include <crm/crm.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/msg_xml.h>
	#include <crm/cib.h>

	int command = 0;
	int ccm_fd = 0;
	gboolean do_quiet = FALSE;

	char *target_uuid = NULL;
	char *target_uname = NULL;
	const char *standby_value = NULL;
	const char *standby_scope = NULL;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},
	{"quiet", 0, 0, 'Q', "\tEssential output only"},

	{"-spacer-", 1, 0, '-', "\nStack:"},
	#if SUPPORT_CMAN
	{"cman", 0, 0, 'c', "\tOnly try connecting to a cman-based cluster"},
	#endif
	#if SUPPORT_COROSYNC
	{"openais", 0, 0, 'A', "\tOnly try connecting to an OpenAIS-based cluster"},
	#endif
	#ifdef SUPPORT_HEARTBEAT
	{"heartbeat", 0, 0, 'H', "Only try connecting to a Heartbeat-based cluster"},
	#endif

	{"-spacer-", 1, 0, '-', "\nCommands:"},
	{"epoch", 0, 0, 'e', "\tDisplay the epoch during which this node joined the cluster"},
	{"quorum", 0, 0, 'q', "\tDisplay a 1 if our partition has quorum, 0 if not"},
	{"list", 0, 0, 'l', "\tDisplay all known members (past and present) of this cluster (Not available for heartbeat clusters)"},
	{"partition", 0, 0, 'p', "Display the members of this partition"},
	{"cluster-id", 0, 0, 'i', "Display this node's cluster id"},
	{"remove", 1, 0, 'R', "(Advanced, AIS-Only) Remove the (stopped) node with the specified nodeid from the cluster"},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{"force", 0, 0, 'f'},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int local_id = 0;

	#if SUPPORT_HEARTBEAT
	# include <ocf/oc_event.h>
	# include <ocf/oc_membership.h>
	# include <clplumbing/cl_uuid.h>

	# define UUID_LEN 16

	oc_ev_t *ccm_token = NULL;
	static void *ccm_library = NULL;
	void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int);

	static gboolean
	read_local_hb_uuid(void)
	{
	cl_uuid_t uuid;
	char *buffer = NULL;
	long start = 0, read_len = 0;

	FILE *input = fopen(UUID_FILE, "r");

	if (input == NULL) {
	crm_info("Could not open UUID file %s\n", UUID_FILE);
	return FALSE;
	}

	/* see how big the file is */
	start = ftell(input);
	fseek(input, 0L, SEEK_END);
	if (UUID_LEN != ftell(input)) {
	fprintf(stderr, "%s must contain exactly %d bytes\n", UUID_FILE, UUID_LEN);
	abort();
	}

	fseek(input, 0L, start);
	if (start != ftell(input)) {
	fprintf(stderr, "fseek not behaving: %ld vs. %ld\n", start, ftell(input));
	exit(2);
	}

	buffer = malloc(50);
	read_len = fread(uuid.uuid, 1, UUID_LEN, input);
	fclose(input);

	if (read_len != UUID_LEN) {
	fprintf(stderr, "Expected and read bytes differ: %d vs. %ld\n", UUID_LEN, read_len);
	exit(3);

	} else if (buffer != NULL) {
	cl_uuid_unparse(&uuid, buffer);
	fprintf(stdout, "%s\n", buffer);
	return TRUE;

	} else {
	fprintf(stderr, "No buffer to unparse\n");
	exit(4);
	}

	free(buffer);
	return FALSE;
	}

	static void
	ccm_age_callback(oc_ed_t event, void cookie, size_t size, const void data)
	{
	int lpc;
	int node_list_size;
	const oc_ev_membership_t oc = (const oc_ev_membership_t )data;

	int (ccm_api_callback_done) (void cookie) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_callback_done", 1);

	node_list_size = oc->m_n_member;
	if (command == 'q') {
	crm_debug("Processing \"%s\" event.",
	event == OC_EV_MS_NEW_MEMBERSHIP ? "NEW MEMBERSHIP" :
	event == OC_EV_MS_NOT_PRIMARY ? "NOT PRIMARY" :
	event == OC_EV_MS_PRIMARY_RESTORED ? "PRIMARY RESTORED" :
	event == OC_EV_MS_EVICTED ? "EVICTED" : "NO QUORUM MEMBERSHIP");
	if (ccm_have_quorum(event)) {
	fprintf(stdout, "1\n");
	} else {
	fprintf(stdout, "0\n");
	}

	} else if (command == 'e') {
	crm_debug("Searching %d members for our birth", oc->m_n_member);
	}
	for (lpc = 0; lpc < node_list_size; lpc++) {
	if (command == 'p') {
	fprintf(stdout, "%s ", oc->m_array[oc->m_memb_idx + lpc].node_uname);

	} else if (command == 'e') {
	int (ccm_api_is_my_nodeid) (const oc_ev_t token, const oc_node_t * node) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_is_my_nodeid", 1);
	if ((*ccm_api_is_my_nodeid) (ccm_token, &(oc->m_array[lpc]))) {
	crm_debug("MATCH: nodeid=%d, uname=%s, born=%d",
	oc->m_array[oc->m_memb_idx + lpc].node_id,
	oc->m_array[oc->m_memb_idx + lpc].node_uname,
	oc->m_array[oc->m_memb_idx + lpc].node_born_on);
	fprintf(stdout, "%d\n", oc->m_array[oc->m_memb_idx + lpc].node_born_on);
	}
	}
	}

	(*ccm_api_callback_done) (cookie);

	if (command == 'p') {
	fprintf(stdout, "\n");
	}
	fflush(stdout);
	exit(0);
	}

	static gboolean
	ccm_age_connect(int *ccm_fd)
	{
	gboolean did_fail = FALSE;
	int ret = 0;

	int (ccm_api_register) (oc_ev_t * token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1);

	int (ccm_api_set_callback) (const oc_ev_t token,
	oc_ev_class_t class,
	oc_ev_callback_t * fn,
	oc_ev_callback_t ** prev_fn) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1);

	void (ccm_api_special) (const oc_ev_t , oc_ev_class_t, int) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1);
	int (ccm_api_activate) (const oc_ev_t token, int *fd) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1);

	crm_debug("Registering with CCM");
	ret = (*ccm_api_register) (&ccm_token);
	if (ret != 0) {
	crm_info("CCM registration failed: %d", ret);
	did_fail = TRUE;
	}

	if (did_fail == FALSE) {
	crm_debug("Setting up CCM callbacks");
	ret = (*ccm_api_set_callback) (ccm_token, OC_EV_MEMB_CLASS, ccm_age_callback, NULL);
	if (ret != 0) {
	crm_warn("CCM callback not set: %d", ret);
	did_fail = TRUE;
	}
	}
	if (did_fail == FALSE) {
	(ccm_api_special) (ccm_token, OC_EV_MEMB_CLASS, 0 /don't care */ );

	crm_debug("Activating CCM token");
	ret = (*ccm_api_activate) (ccm_token, ccm_fd);
	if (ret != 0) {
	crm_warn("CCM Activation failed: %d", ret);
	did_fail = TRUE;
	}
	}

	return !did_fail;
	}

	static gboolean
	try_heartbeat(int command, enum cluster_type_e stack)
	{
	crm_debug("Attempting to process %c command", command);

	if (command == 'i') {
	if (read_local_hb_uuid()) {
	exit(0);
	}

	} else if (ccm_age_connect(&ccm_fd)) {
	int rc = 0;
	fd_set rset;
	int (ccm_api_handle_event) (const oc_ev_t token) =
	find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_handle_event", 1);

	while (1) {

	sleep(1);
	FD_ZERO(&rset);
	FD_SET(ccm_fd, &rset);

	errno = 0;
	rc = select(ccm_fd + 1, &rset, NULL, NULL, NULL);

	if (rc > 0 && (*ccm_api_handle_event) (ccm_token) != 0) {
	crm_err("oc_ev_handle_event failed");
	return FALSE;

	} else if (rc < 0 && errno != EINTR) {
	crm_perror(LOG_ERR, "select failed: %d", rc);
	return FALSE;
	}
	}
	}
	return FALSE;
	}
	#endif

	#if SUPPORT_CMAN
	# include <libcman.h>
	# define MAX_NODES 256
	static gboolean
	try_cman(int command, enum cluster_type_e stack)
	{

	int rc = -1, lpc = 0, node_count = 0;
	cman_node_t node;
	cman_cluster_t cluster;
	cman_handle_t cman_handle = NULL;
	cman_node_t cman_nodes[MAX_NODES];

	memset(&cluster, 0, sizeof(cluster));

	cman_handle = cman_init(NULL);
	if (cman_handle == NULL \|\| cman_is_active(cman_handle) == FALSE) {
	crm_info("Couldn't connect to cman");
	return FALSE;
	}

	switch (command) {
	case 'R':
	fprintf(stderr, "Node removal not supported for cman based clusters\n");
	exit(-EPROTONOSUPPORT);
	break;

	case 'e':
	/* Age makes no sense (yet?) in a cman cluster */
	fprintf(stdout, "1\n");
	break;

	case 'q':
	fprintf(stdout, "%d\n", cman_is_quorate(cman_handle));
	break;

	case 'l':
	case 'p':
	rc = cman_get_nodes(cman_handle, MAX_NODES, &node_count, cman_nodes);
	if (rc != 0) {
	fprintf(stderr, "Couldn't query cman node list: %d %d", rc, errno);
	goto cman_bail;
	}

	for (lpc = 0; lpc < node_count; lpc++) {
	if (command == 'l') {
	printf("%s ", cman_nodes[lpc].cn_name);

	} else if (cman_nodes[lpc].cn_nodeid != 0 && cman_nodes[lpc].cn_member) {
	/* Never allow node ID 0 to be considered a member #315711 */
	printf("%s ", cman_nodes[lpc].cn_name);
	}
	}
	printf("\n");
	break;

	case 'i':
	rc = cman_get_node(cman_handle, CMAN_NODEID_US, &node);
	if (rc != 0) {
	fprintf(stderr, "Couldn't query cman node id: %d %d", rc, errno);
	goto cman_bail;
	}
	fprintf(stdout, "%u\n", node.cn_nodeid);
	break;

	default:
	fprintf(stderr, "Unknown option '%c'\n", command);
	crm_help('?', EX_USAGE);
	}
	cman_finish(cman_handle);
	exit(0);

	cman_bail:
	cman_finish(cman_handle);
	exit(EX_USAGE);
	}
	#endif

	#if HAVE_CONFDB
	static void
	ais_membership_destroy(gpointer user_data)
	{
	crm_err("AIS connection terminated");
	ais_fd_sync = -1;
	exit(1);
	}

	static gint
	member_sort(gconstpointer a, gconstpointer b)
	{
	const crm_node_t *node_a = a;
	const crm_node_t *node_b = b;

	return strcmp(node_a->uname, node_b->uname);
	}

	static void
	crm_add_member(gpointer key, gpointer value, gpointer user_data)
	{
	GList **list = user_data;
	crm_node_t *node = value;

	if (node->uname != NULL) {
	list = g_list_insert_sorted(list, node, member_sort);
	}
	}

	static gboolean
	ais_membership_dispatch(AIS_Message * wrapper, char *data, int sender)
	{
	switch (wrapper->header.id) {
	case crm_class_members:
	case crm_class_notify:
	case crm_class_quorum:
	break;
	default:
	return TRUE;

	break;
	}

	if (command == 'q') {
	if (crm_have_quorum) {
	fprintf(stdout, "1\n");
	} else {
	fprintf(stdout, "0\n");
	}

	} else if (command == 'l') {
	GList *nodes = NULL;
	GListPtr lpc = NULL;

	g_hash_table_foreach(crm_peer_cache, crm_add_member, &nodes);
	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	crm_node_t node = (crm_node_t ) lpc->data;

	fprintf(stdout, "%u %s %s\n", node->id, node->uname, node->state);
	}
	fprintf(stdout, "\n");

	} else if (command == 'p') {
	GList *nodes = NULL;
	GListPtr lpc = NULL;

	g_hash_table_foreach(crm_peer_cache, crm_add_member, &nodes);
	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	crm_node_t node = (crm_node_t ) lpc->data;

	if (node->uname && safe_str_eq(node->state, CRM_NODE_MEMBER)) {
	fprintf(stdout, "%s ", node->uname);
	}
	}
	fprintf(stdout, "\n");
	}

	exit(0);

	return TRUE;
	}
	#endif

	#ifdef SUPPORT_CS_QUORUM
	# include <corosync/quorum.h>
	# include <corosync/cpg.h>
	static int
	node_mcp_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	if (msg) {
	xmlNode *node = NULL;

	crm_log_xml_trace(msg, "message");

	for (node = __xml_first_child(msg); node != NULL; node = __xml_next(node)) {
	const char *uname = crm_element_value(node, "uname");
	if (command == 'l') {
	int id = 0;
	crm_element_value_int(node, "id", &id);
	fprintf(stdout, "%u %s\n", id, uname);

	} else if (command == 'p') {
	fprintf(stdout, "%s ", uname);
	}
	}
	free_xml(msg);

	if (command == 'p') {
	fprintf(stdout, "\n");
	}

	exit(0);
	}

	return 0;
	}

	static void
	node_mcp_destroy(gpointer user_data)
	{
	exit(1);
	}

	static int
	crmd_remove_node_cache(int id)
	{
	int rc = -1;
	char *admin_uuid = NULL;
	crm_ipc_t *conn = crm_ipc_new(CRM_SYSTEM_CRMD, 0);
	xmlNode *cmd = NULL;
	xmlNode *hello = NULL;
	xmlNode *msg_data = NULL;

	if (!conn) {
	goto rm_node_cleanup;
	}

	if (!crm_ipc_connect(conn)) {
	goto rm_node_cleanup;
	}

	admin_uuid = calloc(1, 11);
	snprintf(admin_uuid, 10, "%d", getpid());
	admin_uuid[10] = '\0';

	hello = create_hello_message(admin_uuid, "crm_node", "0", "1");
	rc = crm_ipc_send(conn, hello, NULL, 0);
	if (rc < 0) {
	goto rm_node_cleanup;
	}

	msg_data = create_xml_node(NULL, XML_TAG_OPTIONS);
	crm_xml_add_int(msg_data, XML_ATTR_ID, id);
	cmd = create_request(CRM_OP_RM_NODE_CACHE,
	msg_data,
	NULL,
	CRM_SYSTEM_CRMD,
	"crm_node",
	admin_uuid);

	rc = crm_ipc_send(conn, cmd, NULL, 0);

	rm_node_cleanup:
	if (conn) {
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	}
	free_xml(cmd);
	free_xml(hello);
	free(admin_uuid);
	return rc > 0 ? 0 : rc;
	}

	static gboolean
	try_corosync(int command, enum cluster_type_e stack)
	{
	int rc = 0;
	int quorate = 0;
	uint32_t quorum_type = 0;
	unsigned int nodeid = 0;
	cpg_handle_t c_handle = 0;
	quorum_handle_t q_handle = 0;

	mainloop_io_t *ipc = NULL;
	GMainLoop *amainloop = NULL;

	struct ipc_client_callbacks node_callbacks =
	{
	.dispatch = node_mcp_dispatch,
	.destroy = node_mcp_destroy
	};

	switch (command) {
	case 'R':
	if (crmd_remove_node_cache(atoi(target_uname))) {
	crm_err("Failed to connect to crmd to remove node id %s", target_uname);
	}
	case 'e':
	/* Age makes no sense (yet) in an AIS cluster */
	fprintf(stdout, "1\n");
	exit(0);

	case 'q':
	/* Go direct to the Quorum API */
	rc = quorum_initialize(&q_handle, NULL, &quorum_type);
	if (rc != CS_OK) {
	crm_err("Could not connect to the Quorum API: %d\n", rc);
	return FALSE;
	}

	rc = quorum_getquorate(q_handle, &quorate);
	if (rc != CS_OK) {
	crm_err("Could not obtain the current Quorum API state: %d\n", rc);
	return FALSE;
	}

	if (quorate) {
	fprintf(stdout, "1\n");
	} else {
	fprintf(stdout, "0\n");
	}
	quorum_finalize(q_handle);
	exit(0);

	case 'i':
	/* Go direct to the CPG API */
	rc = cpg_initialize(&c_handle, NULL);
	if (rc != CS_OK) {
	crm_err("Could not connect to the Cluster Process Group API: %d\n", rc);
	return FALSE;
	}

	rc = cpg_local_get(c_handle, &nodeid);
	if (rc != CS_OK) {
	crm_err("Could not get local node id from the CPG API");
	return FALSE;
	}

	fprintf(stdout, "%u\n", nodeid);
	cpg_finalize(c_handle);
	exit(0);

	case 'l':
	case 'p':
	/* Go to pacemakerd */
	amainloop = g_main_new(FALSE);
	- ipc = mainloop_add_ipc_client(CRM_SYSTEM_MCP, 0, NULL, &node_callbacks);
	+ ipc = mainloop_add_ipc_client(CRM_SYSTEM_MCP, G_PRIORITY_DEFAULT, 0, NULL, &node_callbacks);
	if(ipc != NULL) {
	xmlNode *poke = create_xml_node(NULL, "poke");
	crm_ipc_send(mainloop_get_ipc_client(ipc), poke, NULL, 0);
	free_xml(poke);
	g_main_run(amainloop);
	}
	break;
	}
	return FALSE;
	}
	#endif

	#if HAVE_CONFDB
	static gboolean
	try_openais(int command, enum cluster_type_e stack)
	{
	if (init_ais_connection_once
	(ais_membership_dispatch, ais_membership_destroy, NULL, NULL, &local_id)) {

	GMainLoop *amainloop = NULL;

	switch (command) {
	case 'R':
	send_ais_text(crm_class_rmpeer, target_uname, TRUE, NULL, crm_msg_ais);
	exit(0);

	case 'e':
	/* Age makes no sense (yet) in an AIS cluster */
	fprintf(stdout, "1\n");
	exit(0);

	case 'q':
	send_ais_text(crm_class_quorum, NULL, TRUE, NULL, crm_msg_ais);
	break;

	case 'l':
	case 'p':
	crm_info("Requesting the list of configured nodes");
	send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais);
	break;

	case 'i':
	printf("%u\n", local_id);
	exit(0);

	default:
	fprintf(stderr, "Unknown option '%c'\n", command);
	crm_help('?', EX_USAGE);
	}
	amainloop = g_main_new(FALSE);
	g_main_run(amainloop);
	}
	return FALSE;
	}
	#endif

	int set_cluster_type(enum cluster_type_e type);

	int
	main(int argc, char **argv)
	{
	int flag = 0;
	int argerr = 0;
	gboolean force_flag = FALSE;
	gboolean dangerous_cmd = FALSE;
	enum cluster_type_e try_stack = pcmk_cluster_unknown;

	int option_index = 0;

	crm_peer_init();
	crm_log_cli_init("crm_node");
	crm_set_options(NULL, "command [options]", long_options,
	"Tool for displaying low-level node information");

	while (flag >= 0) {
	flag = crm_get_option(argc, argv, &option_index);
	switch (flag) {
	case -1:
	break;
	case 'V':
	crm_bump_log_level();
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'Q':
	do_quiet = TRUE;
	break;
	case 'H':
	set_cluster_type(pcmk_cluster_heartbeat);
	break;
	case 'A':
	set_cluster_type(pcmk_cluster_classic_ais);
	break;
	case 'C':
	set_cluster_type(pcmk_cluster_corosync);
	break;
	case 'c':
	set_cluster_type(pcmk_cluster_cman);
	break;
	case 'f':
	force_flag = TRUE;
	break;
	case 'R':
	dangerous_cmd = TRUE;
	command = flag;
	target_uname = optarg;
	break;
	case 'p':
	case 'e':
	case 'q':
	case 'i':
	case 'l':
	command = flag;
	break;
	default:
	++argerr;
	break;
	}
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	if (dangerous_cmd && force_flag == FALSE) {
	fprintf(stderr, "The supplied command is considered dangerous."
	" To prevent accidental destruction of the cluster,"
	" the --force flag is required in order to proceed.\n");
	fflush(stderr);
	exit(EX_USAGE);
	}

	try_stack = get_cluster_type();
	crm_debug("Attempting to process -%c command for cluster type: %s", command,
	name_for_cluster_type(try_stack));

	#if SUPPORT_CMAN
	if (try_stack == pcmk_cluster_cman) {
	try_cman(command, try_stack);
	}
	#endif

	#ifdef SUPPORT_CS_QUORUM
	if (try_stack == pcmk_cluster_corosync) {
	try_corosync(command, try_stack);
	}
	#endif

	#if HAVE_CONFDB
	/* Only an option if we're using the plugins */
	if (try_stack == pcmk_cluster_classic_ais) {
	try_openais(command, try_stack);
	}
	#endif

	#if SUPPORT_HEARTBEAT
	if (try_stack == pcmk_cluster_heartbeat) {
	try_heartbeat(command, try_stack);
	}
	#endif

	return (1);
	}
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index f6462a67d9..ff441b8039 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,1926 +1,1926 @@

	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>

	#include <crm/cib.h>
	#include <crm/attrd.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>

	gboolean do_force = FALSE;
	gboolean BE_QUIET = FALSE;
	const char *attr_set_type = XML_TAG_ATTR_SETS;
	char *host_id = NULL;
	const char *rsc_id = NULL;
	const char *host_uname = NULL;
	const char *prop_name = NULL;
	const char *prop_value = NULL;
	const char *rsc_type = NULL;
	const char *prop_id = NULL;
	const char *prop_set = NULL;
	char *move_lifetime = NULL;
	char rsc_cmd = 'L';
	const char *rsc_long_cmd = NULL;
	char *our_pid = NULL;
	crm_ipc_t *crmd_channel = NULL;
	char *xml_file = NULL;
	int cib_options = cib_sync_call;
	int crmd_replies_needed = 0;
	GMainLoop *mainloop = NULL;

	extern void cleanup_alloc_calculations(pe_working_set_t * data_set);

	#define CMD_ERR(fmt, args...) do { \
	crm_warn(fmt, ##args); \
	fprintf(stderr, fmt, ##args); \
	} while(0)

	#define message_timeout_ms 60*1000

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	fprintf(stderr, "No messages received in %d seconds.. aborting\n",
	(int)message_timeout_ms / 1000);
	crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
	exit(-1);
	}

	static void
	resource_ipc_connection_destroy(gpointer user_data)
	{
	crm_info("Connection to CRMd was terminated");
	exit(1);
	}

	static void
	start_mainloop(void)
	{
	mainloop = g_main_new(FALSE);
	crmd_replies_needed++; /* The welcome message */
	fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed);
	crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed);

	g_timeout_add(message_timeout_ms, resource_ipc_timeout, NULL);
	g_main_run(mainloop);
	}

	static int
	resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	fprintf(stderr, ".");
	crm_log_xml_trace(msg, "[inbound]");

	crmd_replies_needed--;
	if (crmd_replies_needed == 0) {
	fprintf(stderr, " OK\n");
	crm_debug("Got all the replies we expected");
	crm_xml_cleanup();
	exit(0);
	}

	free_xml(msg);
	return 0;
	}

	struct ipc_client_callbacks crm_callbacks =
	{
	.dispatch = resource_ipc_callback,
	.destroy = resource_ipc_connection_destroy,
	};

	static int
	do_find_resource(const char rsc, resource_t the_rsc, pe_working_set_t * data_set)
	{
	int found = 0;
	GListPtr lpc = NULL;

	if (the_rsc == NULL) {
	the_rsc = pe_find_resource(data_set->resources, rsc);
	}

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	if (the_rsc->variant > pe_clone) {
	GListPtr gIter = the_rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	found += do_find_resource(rsc, gIter->data, data_set);
	}
	return found;
	}

	for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
	node_t node = (node_t ) lpc->data;

	crm_trace("resource %s is running on: %s", rsc, node->details->uname);
	if (BE_QUIET) {
	fprintf(stdout, "%s\n", node->details->uname);
	} else {
	const char *state = "";

	if (the_rsc->variant == pe_native && the_rsc->role == RSC_ROLE_MASTER) {
	state = "Master";
	}
	fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
	}

	found++;
	}

	if (BE_QUIET == FALSE && found == 0) {
	fprintf(stderr, "resource %s is NOT running\n", rsc);
	}

	return 0;
	}

	#define cons_string(x) x?x:"NA"
	static void
	print_cts_constraints(pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	xmlNode *lifetime = NULL;
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	for (xml_obj = __xml_first_child(cib_constraints); xml_obj != NULL;
	xml_obj = __xml_next(xml_obj)) {
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);

	if (id == NULL) {
	continue;
	}

	lifetime = first_named_child(xml_obj, "lifetime");

	if (test_ruleset(lifetime, NULL, data_set->now) == FALSE) {
	continue;
	}

	if (safe_str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj))) {
	printf("Constraint %s %s %s %s %s %s %s\n",
	crm_element_name(xml_obj),
	cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
	cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));

	} else if (safe_str_eq(XML_CONS_TAG_RSC_LOCATION, crm_element_name(xml_obj))) {
	/* unpack_rsc_location(xml_obj, data_set); */
	}
	}
	}

	static void
	print_cts_rsc(resource_t * rsc)
	{
	GListPtr lpc = NULL;
	const char *host = NULL;
	gboolean needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (safe_str_eq(rclass, "stonith")) {
	xmlNode *op = NULL;

	needs_quorum = FALSE;

	for (op = __xml_first_child(rsc->ops_xml); op != NULL; op = __xml_next(op)) {
	if (crm_str_eq((const char *)op->name, "op", TRUE)) {
	const char *name = crm_element_value(op, "name");

	if (safe_str_neq(name, CRMD_ACTION_START)) {
	const char *value = crm_element_value(op, "requires");

	if (safe_str_eq(value, "nothing")) {
	needs_quorum = FALSE;
	}
	break;
	}
	}
	}
	}

	if (rsc->running_on != NULL && g_list_length(rsc->running_on) == 1) {
	node_t *tmp = rsc->running_on->data;

	host = tmp->details->uname;
	}

	printf("Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx\n",
	crm_element_name(rsc->xml), rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	print_cts_rsc(child);
	}
	}

	static void
	print_raw_rsc(resource_t * rsc)
	{
	GListPtr lpc = NULL;
	GListPtr children = rsc->children;

	if (children == NULL) {
	printf("%s\n", rsc->id);
	}

	for (lpc = children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	print_raw_rsc(child);
	}
	}

	static int
	do_find_resource_list(pe_working_set_t * data_set, gboolean raw)
	{
	int found = 0;

	GListPtr lpc = NULL;

	for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	if (is_set(rsc->flags, pe_rsc_orphan)
	&& rsc->fns->active(rsc, TRUE) == FALSE) {
	continue;
	}
	rsc->fns->print(rsc, NULL, pe_print_printf \| pe_print_rsconly, stdout);
	found++;
	}

	if (found == 0) {
	printf("NO resources configured\n");
	return -ENXIO;
	}

	return 0;
	}

	static resource_t *
	find_rsc_or_clone(const char rsc, pe_working_set_t data_set)
	{
	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);

	if (the_rsc == NULL) {
	char *as_clone = crm_concat(rsc, "0", ':');

	the_rsc = pe_find_resource(data_set->resources, as_clone);
	free(as_clone);
	}
	return the_rsc;
	}

	static int
	dump_resource(const char rsc, pe_working_set_t data_set, gboolean expanded)
	{
	char *rsc_xml = NULL;
	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);

	if (the_rsc == NULL) {
	return -ENXIO;
	}
	the_rsc->fns->print(the_rsc, NULL, pe_print_printf, stdout);

	if (expanded) {
	rsc_xml = dump_xml_formatted(the_rsc->xml);
	} else {
	if (the_rsc->orig_xml) {
	rsc_xml = dump_xml_formatted(the_rsc->orig_xml);
	} else {
	rsc_xml = dump_xml_formatted(the_rsc->xml);
	}
	}

	fprintf(stdout, "%sxml:\n%s\n", expanded ? "" : "raw ", rsc_xml);

	free(rsc_xml);

	return 0;
	}

	static int
	dump_resource_attr(const char rsc, const char attr, pe_working_set_t * data_set)
	{
	int rc = -ENXIO;
	node_t *current = NULL;
	GHashTable *params = NULL;
	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
	const char *value = NULL;

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	if (g_list_length(the_rsc->running_on) == 1) {
	current = the_rsc->running_on->data;

	} else if (g_list_length(the_rsc->running_on) > 1) {
	CMD_ERR("%s is active on more than one node,"
	" returning the default value for %s\n", the_rsc->id, crm_str(value));
	}

	params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
	get_rsc_attributes(params, the_rsc, current, data_set);
	} else if (safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
	get_meta_attributes(params, the_rsc, current, data_set);
	} else {
	unpack_instance_attributes(data_set->input, the_rsc->xml, XML_TAG_UTILIZATION, NULL,
	params, NULL, FALSE, data_set->now);
	}

	crm_debug("Looking up %s in %s", attr, the_rsc->id);
	value = g_hash_table_lookup(params, attr);
	if (value != NULL) {
	fprintf(stdout, "%s\n", value);
	rc = 0;
	}

	g_hash_table_destroy(params);
	return rc;
	}

	static int
	find_resource_attr(cib_t * the_cib, const char attr, const char rsc, const char *set_type,
	const char set_name, const char attr_id, const char attr_name, char *value)
	{
	int offset = 0;
	static int xpath_max = 1024;
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	char *xpath_string = NULL;

	CRM_ASSERT(value != NULL);
	*value = NULL;

	xpath_string = calloc(1, xpath_max);
	offset +=
	snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);

	if (set_type) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s", set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "]");

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);

	if (rc != pcmk_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = -EINVAL;
	printf("Multiple attributes match name=%s\n", attr_name);

	for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
	printf(" Value: %s \t(id=%s)\n",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	} else {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	bail:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	static int
	set_resource_attr(const char rsc_id, const char attr_set, const char *attr_id,
	const char attr_name, const char attr_value,
	cib_t * cib, pe_working_set_t * data_set)
	{
	int rc = pcmk_ok;

	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;

	gboolean use_attributes_tag = FALSE;
	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);

	if (rsc == NULL) {
	return -ENXIO;
	}

	if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_ok) {
	printf("WARNING: There is already a meta attribute called %s (id=%s)\n", attr_name,
	local_attr_id);
	}
	}
	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == pcmk_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != -ENXIO) {
	free(local_attr_id);
	return rc;

	} else {
	const char *value = NULL;
	xmlNode *cib_top = NULL;
	const char *tag = crm_element_name(rsc->xml);

	rc = cib->cmds->query(cib, "/cib", &cib_top,
	cib_sync_call \| cib_scope_local \| cib_xpath \| cib_no_children);
	value = crm_element_value(cib_top, "ignore_dtd");
	if (value != NULL) {
	use_attributes_tag = TRUE;

	} else {
	value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
	if (value && strstr(value, "-0.6")) {
	use_attributes_tag = TRUE;
	}
	}
	free_xml(cib_top);

	if (attr_set == NULL) {
	local_attr_set = crm_concat(rsc_id, attr_set_type, '-');
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_concat(attr_set, attr_name, '-');
	attr_id = local_attr_id;
	}

	if (use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
	tag = "master_slave"; /* use the old name */
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, rsc_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);

	if (use_attributes_tag) {
	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
	}
	}

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	free_xml(xml_top);
	free(local_attr_id);
	free(local_attr_set);
	return rc;
	}

	static int
	delete_resource_attr(const char rsc_id, const char attr_set, const char *attr_id,
	const char attr_name, cib_t cib, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	int rc = pcmk_ok;
	char *local_attr_id = NULL;
	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);

	if (rsc == NULL) {
	return -ENXIO;
	}

	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == -ENXIO) {
	return pcmk_ok;

	} else if (rc != pcmk_ok) {
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);

	crm_log_xml_debug(xml_obj, "Delete");

	rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);

	if (rc == pcmk_ok) {
	printf("Deleted %s option: id=%s%s%s%s%s\n", rsc_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free_xml(xml_obj);
	free(local_attr_id);
	return rc;
	}

	static int
	dump_resource_prop(const char rsc, const char attr, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	value = crm_element_value(the_rsc->xml, attr);

	if (value != NULL) {
	fprintf(stdout, "%s\n", value);
	return 0;
	}
	return -ENXIO;
	}

	static int
	send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
	const char host_uname, const char rsc_id,
	gboolean only_failed, pe_working_set_t * data_set)
	{
	char *key = NULL;
	int rc = -ECOMM;
	xmlNode *cmd = NULL;
	xmlNode *xml_rsc = NULL;
	const char *value = NULL;
	xmlNode *params = NULL;
	xmlNode *msg_data = NULL;
	resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	CMD_ERR("Resource %s not found\n", rsc_id);
	return -ENXIO;

	} else if (rsc->variant != pe_native) {
	CMD_ERR("We can only process primitive resources, not %s\n", rsc_id);
	return -EINVAL;

	} else if (host_uname == NULL) {
	CMD_ERR("Please supply a hostname with -H\n");
	return -EINVAL;
	}

	key = crm_concat("0:0:crm-resource", our_pid, '-');

	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
	free(key);

	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
	if (rsc->clone_name) {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
	crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);

	} else {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
	}

	value = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, value);
	if (value == NULL) {
	CMD_ERR("%s has no type! Aborting...\n", rsc_id);
	return -ENXIO;
	}

	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, value);
	if (value == NULL) {
	CMD_ERR("%s has no class! Aborting...\n", rsc_id);
	return -ENXIO;
	}

	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, value);

	params = create_xml_node(msg_data, XML_TAG_ATTRS);
	crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	crm_xml_add(params, key, "60000"); /* 1 minute */
	free(key);

	cmd = create_request(op, msg_data, host_uname, CRM_SYSTEM_CRMD, crm_system_name, our_pid);

	/* crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
	free_xml(msg_data);

	if (crm_ipc_send(crmd_channel, cmd, NULL, 0) > 0) {
	rc = 0;

	} else {
	CMD_ERR("Could not send %s op to the crmd", op);
	rc = -ENOTCONN;
	}

	free_xml(cmd);
	return rc;
	}

	static int
	delete_lrm_rsc(crm_ipc_t * crmd_channel, const char *host_uname,
	resource_t * rsc, pe_working_set_t * data_set)
	{
	int rc = pcmk_ok;

	if (rsc == NULL) {
	return -ENXIO;

	} else if (rsc->children) {
	GListPtr lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	delete_lrm_rsc(crmd_channel, host_uname, child, data_set);
	}
	return pcmk_ok;

	} else if (host_uname == NULL) {
	GListPtr lpc = NULL;

	for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
	node_t node = (node_t ) lpc->data;

	if (node->details->online) {
	delete_lrm_rsc(crmd_channel, node->details->uname, rsc, data_set);
	}
	}

	return pcmk_ok;
	}

	printf("Cleaning up %s on %s\n", rsc->id, host_uname);
	rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
	if (rc == pcmk_ok) {
	char *attr_name = NULL;
	const char *id = rsc->id;

	if (rsc->clone_name) {
	id = rsc->clone_name;
	}

	attr_name = crm_concat("fail-count", id, '-');
	attrd_update_delegate(NULL, 'D', host_uname, attr_name, NULL, XML_CIB_TAG_STATUS, NULL, NULL, NULL);
	free(attr_name);
	}
	return rc;
	}

	static int
	fail_lrm_rsc(crm_ipc_t * crmd_channel, const char *host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	crm_warn("Failing: %s", rsc_id);
	return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
	}

	static int
	refresh_lrm(crm_ipc_t * crmd_channel, const char *host_uname)
	{
	xmlNode *cmd = NULL;
	int rc = -ECOMM;

	cmd = create_request(CRM_OP_LRM_REFRESH, NULL, host_uname,
	CRM_SYSTEM_CRMD, crm_system_name, our_pid);

	if (crm_ipc_send(crmd_channel, cmd, NULL, 0) > 0) {
	rc = 0;
	}
	free_xml(cmd);
	return rc;
	}

	static int
	move_resource(const char *rsc_id,
	const char existing_node, const char preferred_node, cib_t * cib_conn)
	{
	char *later_s = NULL;
	int rc = pcmk_ok;
	char *id = NULL;
	xmlNode *rule = NULL;
	xmlNode *expr = NULL;
	xmlNode *constraints = NULL;
	xmlNode *fragment = NULL;

	xmlNode *can_run = NULL;
	xmlNode *dont_run = NULL;

	fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
	constraints = fragment;

	id = crm_concat("cli-prefer", rsc_id, '-');
	can_run = create_xml_node(NULL, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(can_run, XML_ATTR_ID, id);
	free(id);

	id = crm_concat("cli-standby", rsc_id, '-');
	dont_run = create_xml_node(NULL, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(dont_run, XML_ATTR_ID, id);
	free(id);

	if (move_lifetime) {
	char *life = strdup(move_lifetime);
	char *life_mutable = life;

	ha_time_t *now = NULL;
	ha_time_t *later = NULL;
	ha_time_t *duration = parse_time_duration(&life_mutable);

	if (duration == NULL) {
	CMD_ERR("Invalid duration specified: %s\n", move_lifetime);
	CMD_ERR("Please refer to"
	" http://en.wikipedia.org/wiki/ISO_8601#Duration"
	" for examples of valid durations\n");
	free(life);
	return -EINVAL;
	}
	now = new_ha_date(TRUE);
	later = add_time(now, duration);
	log_date(LOG_INFO, "now ", now, ha_log_date \| ha_log_time);
	log_date(LOG_INFO, "later ", later, ha_log_date \| ha_log_time);
	log_date(LOG_INFO, "duration", duration, ha_log_date \| ha_log_time \| ha_log_local);
	later_s = date_to_string(later, ha_log_date \| ha_log_time);
	printf("Migration will take effect until: %s\n", later_s);

	free_ha_date(duration);
	free_ha_date(later);
	free_ha_date(now);
	free(life);
	}

	if (existing_node == NULL) {
	crm_log_xml_notice(can_run, "Deleting");
	rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_CONSTRAINTS, dont_run, cib_options);
	if (rc == -ENXIO) {
	rc = pcmk_ok;

	} else if (rc != pcmk_ok) {
	goto bail;
	}

	} else {
	if (BE_QUIET == FALSE) {
	fprintf(stderr,
	"WARNING: Creating rsc_location constraint '%s'"
	" with a score of -INFINITY for resource %s"
	" on %s.\n", ID(dont_run), rsc_id, existing_node);
	CMD_ERR("\tThis will prevent %s from running"
	" on %s until the constraint is removed using"
	" the 'crm_resource -U' command or manually"
	" with cibadmin\n", rsc_id, existing_node);
	CMD_ERR("\tThis will be the case even if %s is"
	" the last node in the cluster\n", existing_node);
	CMD_ERR("\tThis message can be disabled with -Q\n");
	}

	crm_xml_add(dont_run, "rsc", rsc_id);

	rule = create_xml_node(dont_run, XML_TAG_RULE);
	expr = create_xml_node(rule, XML_TAG_EXPRESSION);
	id = crm_concat("cli-standby-rule", rsc_id, '-');
	crm_xml_add(rule, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(rule, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);
	crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");

	id = crm_concat("cli-standby-expr", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
	crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
	crm_xml_add(expr, XML_EXPR_ATTR_VALUE, existing_node);
	crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");

	if (later_s) {
	expr = create_xml_node(rule, "date_expression");
	id = crm_concat("cli-standby-lifetime-end", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, "operation", "lt");
	crm_xml_add(expr, "end", later_s);
	}

	add_node_copy(constraints, dont_run);
	}

	if (preferred_node == NULL) {
	crm_log_xml_notice(can_run, "Deleting");
	rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_CONSTRAINTS, can_run, cib_options);
	if (rc == -ENXIO) {
	rc = pcmk_ok;

	} else if (rc != pcmk_ok) {
	goto bail;
	}

	} else {
	crm_xml_add(can_run, "rsc", rsc_id);

	rule = create_xml_node(can_run, XML_TAG_RULE);
	expr = create_xml_node(rule, XML_TAG_EXPRESSION);
	id = crm_concat("cli-prefer-rule", rsc_id, '-');
	crm_xml_add(rule, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(rule, XML_RULE_ATTR_SCORE, INFINITY_S);
	crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");

	id = crm_concat("cli-prefer-expr", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
	crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
	crm_xml_add(expr, XML_EXPR_ATTR_VALUE, preferred_node);
	crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");

	if (later_s) {
	expr = create_xml_node(rule, "date_expression");
	id = crm_concat("cli-prefer-lifetime-end", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, "operation", "lt");
	crm_xml_add(expr, "end", later_s);
	}

	add_node_copy(constraints, can_run);
	}

	if (preferred_node != NULL \|\| existing_node != NULL) {
	crm_log_xml_notice(fragment, "CLI Update");
	rc = cib_conn->cmds->update(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
	}

	bail:
	free_xml(fragment);
	free_xml(dont_run);
	free_xml(can_run);
	free(later_s);
	return rc;
	}

	static int
	list_resource_operations(const char rsc_id, const char host_uname, gboolean active,
	pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	int opts = pe_print_printf \| pe_print_rsconly \| pe_print_suppres_nl;
	GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);
	GListPtr lpc = NULL;

	for (lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	const char *last = crm_element_value(xml_op, "last_run");
	const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
	const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	int status = crm_parse_int(status_s, "0");

	rsc = pe_find_resource(data_set->resources, op_rsc);
	rsc->fns->print(rsc, "", opts, stdout);

	fprintf(stdout, ": %s (node=%s, call=%s, rc=%s",
	op_key ? op_key : ID(xml_op),
	crm_element_value(xml_op, XML_ATTR_UNAME),
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	crm_element_value(xml_op, XML_LRM_ATTR_RC));
	if (last) {
	time_t run_at = crm_parse_int(last, "0");

	fprintf(stdout, ", last-run=%s, exec=%sms\n",
	ctime(&run_at), crm_element_value(xml_op, "exec_time"));
	}
	fprintf(stdout, "): %s\n", services_lrm_status_str(status));
	}
	return pcmk_ok;
	}

	#include "../pengine/pengine.h"

	static void
	show_location(resource_t * rsc, const char *prefix)
	{
	GListPtr lpc = NULL;
	GListPtr list = rsc->rsc_location;
	int offset = 0;

	if (prefix) {
	offset = strlen(prefix) - 2;
	}

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	rsc_to_node_t cons = (rsc_to_node_t ) lpc->data;

	GListPtr lpc2 = NULL;

	for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
	node_t node = (node_t ) lpc2->data;
	char *score = score2char(node->weight);

	fprintf(stdout, "%s: Node %-*s (score=%s, id=%s)\n",
	prefix ? prefix : " ", 71 - offset, node->details->uname, score, cons->id);
	free(score);
	}
	}
	}

	static void
	show_colocation(resource_t * rsc, gboolean dependants, gboolean recursive, int offset)
	{
	char *prefix = NULL;
	GListPtr lpc = NULL;
	GListPtr list = rsc->rsc_cons;

	prefix = calloc(1, (offset * 4) + 1);
	memset(prefix, ' ', offset * 4);

	if (dependants) {
	list = rsc->rsc_cons_lhs;
	}

	if (is_set(rsc->flags, pe_rsc_allocating)) {
	/* Break colocation loops */
	printf("loop %s\n", rsc->id);
	free(prefix);
	return;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) lpc->data;

	char *score = NULL;
	resource_t *peer = cons->rsc_rh;

	if (dependants) {
	peer = cons->rsc_lh;
	}

	if (is_set(peer->flags, pe_rsc_allocating)) {
	if (dependants == FALSE) {
	fprintf(stdout, "%s%-s (id=%s - loop)\n", prefix, 80 - (4 offset), peer->id,
	cons->id);
	}
	continue;
	}

	if (dependants && recursive) {
	show_colocation(peer, dependants, recursive, offset + 1);
	}

	score = score2char(cons->score);
	if (cons->role_rh > RSC_ROLE_STARTED) {
	fprintf(stdout, "%s%-s (score=%s, %s role=%s, id=%s)\n", prefix, 80 - (4 offset),
	peer->id, score, dependants ? "needs" : "with", role2text(cons->role_rh),
	cons->id);
	} else {
	fprintf(stdout, "%s%-s (score=%s, id=%s)\n", prefix, 80 - (4 offset),
	peer->id, score, cons->id);
	}
	show_location(peer, prefix);
	free(score);

	if (!dependants && recursive) {
	show_colocation(peer, dependants, recursive, offset + 1);
	}
	}
	free(prefix);
	}

	static GHashTable *
	generate_resource_params(resource_t rsc, pe_working_set_t data_set)
	{
	int rc = 0;
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;

	if (!rsc) {
	crm_err("Resource does not exist in config");
	rc = -1;
	return NULL;
	}

	params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	combined = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	get_rsc_attributes(params, rsc, NULL/* TODO: Pass in local node */, data_set);
	get_meta_attributes(meta, rsc, NULL/* TODO: Pass in local node */, data_set);

	if (params) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	g_hash_table_destroy(params);
	}

	if (meta) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}


	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\t\tThis text"},
	{"version", 0, 0, '$', "\t\tVersion information" },
	{"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
	{"quiet", 0, 0, 'Q', "\t\tPrint only the value on stdout\n"},

	{"resource", 1, 0, 'r', "\tResource ID" },

	{"-spacer-",1, 0, '-', "\nQueries:"},
	{"list", 0, 0, 'L', "\t\tList all cluster resources"},
	{"list-raw", 0, 0, 'l', "\tList the IDs of all instantiated resources (no groups/clones/...)"},
	{"list-cts", 0, 0, 'c', NULL, 1},
	{"list-operations", 0, 0, 'O', "\tList active resource operations. Optionally filtered by resource (-r) and/or node (-N)"},
	{"list-all-operations", 0, 0, 'o', "List all resource operations. Optionally filtered by resource (-r) and/or node (-N)\n"},

	{"list-standards", 0, 0, 0, "\tList supported standards"},
	{"list-ocf-providers", 0, 0, 0, "List all available OCF providers"},
	{"list-agents", 1, 0, 0, "List all agents available for the named standard and/or provider."},
	{"list-ocf-alternatives", 1, 0, 0, "List all available providers for the named OCF agent\n"},
	{"show-metadata", 1, 0, 0, "Show the metadata for the named class:provider:agent"},

	{"query-xml", 0, 0, 'q', "\tQuery the definition of a resource (template expanded)"},
	{"query-xml-raw", 0, 0, 'w', "\tQuery the definition of a resource (raw xml)"},
	{"locate", 0, 0, 'W', "\t\tDisplay the current location(s) of a resource"},
	{"stack", 0, 0, 'A', "\t\tDisplay the prerequisites and dependents of a resource"},
	{"constraints",0, 0, 'a', "\tDisplay the (co)location constraints that apply to a resource"},

	{"-spacer-", 1, 0, '-', "\nCommands:"},
	{"set-parameter", 1, 0, 'p', "Set the named parameter for a resource. See also -m, --meta"},
	{"get-parameter", 1, 0, 'g', "Display the named parameter for a resource. See also -m, --meta"},
	{"delete-parameter",1, 0, 'd', "Delete the named parameter for a resource. See also -m, --meta"},
	{"get-property", 1, 0, 'G', "Display the 'class', 'type' or 'provider' of a resource", 1},
	{"set-property", 1, 0, 'S', "(Advanced) Set the class, type or provider of a resource", 1},
	{"move", 0, 0, 'M',
	"\t\tMove a resource from its current location, optionally specifying a destination (-N) and/or a period for which it should take effect (-u)"
	"\n\t\t\t\tIf -N is not specified, the cluster will force the resource to move by creating a rule for the current location and a score of -INFINITY"
	"\n\t\t\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U"},
	{"un-move", 0, 0, 'U', "\t\tRemove all constraints created by a move command"},

	{"-spacer-", 1, 0, '-', "\nAdvanced Commands:"},
	{"delete", 0, 0, 'D', "\t\t(Advanced) Delete a resource from the CIB"},
	{"fail", 0, 0, 'F', "\t\t(Advanced) Tell the cluster this resource has failed"},
	{"refresh", 0, 0, 'R', "\t\t(Advanced) Refresh the CIB from the LRM"},
	{"cleanup", 0, 0, 'C', "\t\t(Advanced) Delete a resource from the LRM"},
	{"reprobe", 0, 0, 'P', "\t\t(Advanced) Re-check for resources started outside of the CRM\n"},
	{"force-stop", 0, 0, 0, "\t(Advanced) Bypass the cluster and stop a resource on the local node"},
	{"force-start",0, 0, 0, "\t(Advanced) Bypass the cluster and start a resource on the local node"},
	{"force-check",0, 0, 0, "\t(Advanced) Bypass the cluster and check the state of a resource on the local node\n"},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{"node", 1, 0, 'N', "\tHost uname"},
	{"resource-type", 1, 0, 't', "Resource type (primitive, clone, group, ...)"},
	{"parameter-value", 1, 0, 'v', "Value to use with -p, -g or -d"},
	{"lifetime", 1, 0, 'u', "\tLifespan of migration constraints\n"},
	{"meta", 0, 0, 'm', "\t\tModify a resource's configuration option rather than one which is passed to the resource agent script. For use with -p, -g, -d"},
	{"utilization", 0, 0, 'z', "\tModify a resource's utilization attribute. For use with -p, -g, -d"},
	{"set-name", 1, 0, 's', "\t(Advanced) ID of the instance_attributes object to change"},
	{"nvpair", 1, 0, 'i', "\t(Advanced) ID of the nvpair object to change/delete"},
	{"force", 0, 0, 'f', "\n" /* Is this actually true anymore?
	"\t\tForce the resource to move by creating a rule for the current location and a score of -INFINITY"
	"\n\t\tThis should be used if the resource's stickiness and constraint scores total more than INFINITY (Currently 100,000)"
	"\n\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U or the --lifetime duration expires\n"*/ },

	{"xml-file", 1, 0, 'x', NULL, 1},\

	/* legacy options */
	{"host-uname", 1, 0, 'H', NULL, 1},
	{"migrate", 0, 0, 'M', NULL, 1},
	{"un-migrate", 0, 0, 'U', NULL, 1},

	{"-spacer-", 1, 0, '-', "\nExamples:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "List the configured resources:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "List the available OCF agents:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list-agents ocf", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Display the current location of 'myResource':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --locate", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Move 'myResource' to another machine:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --move", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Move 'myResource' to a specific machine:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --un-move", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Tell the cluster that 'myResource' failed:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --fail", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Stop a 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
	{"-spacer-", 1, 0, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
	{"-spacer-", 1, 0, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	const char *longname = NULL;
	pe_working_set_t data_set;
	xmlNode *cib_xml_copy = NULL;

	cib_t *cib_conn = NULL;
	lrmd_t *lrmd_conn = NULL;
	int rc = pcmk_ok;

	int option_index = 0;
	int argerr = 0;
	int flag;

	crm_log_cli_init("crm_resource");
	crm_set_options(NULL, "(query\|command) [options]", long_options,
	"Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");

	if (argc < 2) {
	crm_help('?', EX_USAGE);
	}

	lrmd_conn = lrmd_api_new();

	while (1) {
	flag = crm_get_option_long(argc, argv, &option_index, &longname);
	if (flag == -1)
	break;

	switch (flag) {
	case 0:
	if(safe_str_eq("force-stop", longname)
	\|\| safe_str_eq("force-start", longname)
	\|\| safe_str_eq("force-check", longname)) {
	rsc_cmd = flag;
	rsc_long_cmd = longname;

	} else if(safe_str_eq("list-ocf-providers", longname)
	\|\| safe_str_eq("list-ocf-alternatives", longname)
	\|\| safe_str_eq("list-standards", longname)) {
	const char *text = NULL;
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	if(safe_str_eq("list-ocf-providers", longname) \|\| safe_str_eq("list-ocf-alternatives", longname)) {
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
	text = "OCF providers";

	} else if(safe_str_eq("list-standards", longname)) {
	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
	text = "standards";
	}

	if (rc > 0) {
	rc = 0;
	for (iter = list; iter != NULL; iter = iter->next) {
	rc++;
	printf("%s\n", iter->val);
	}
	lrmd_list_freeall(list);
	if(optarg) {
	fprintf(stderr, "%d %s found for %s\n", rc, text, optarg);
	} else {
	fprintf(stderr, "%d %s found\n", rc, text);
	}
	return 0;

	} else {
	if(optarg) {
	fprintf(stderr, "No %s found for %s\n", text, optarg);
	} else {
	fprintf(stderr, "No %s found\n", text);
	}
	return -1;
	}

	} else if(safe_str_eq("show-metadata", longname)) {
	char standard[512];
	char provider[512];
	char type[512];
	char *metadata = NULL;

	rc = sscanf(optarg, "%[^:]:%[^:]:%s", standard, provider, type);
	if(rc == 3) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, provider, type, &metadata, 0);
	if(metadata) {
	printf("%s\n", metadata);
	return 0;
	}
	fprintf(stderr, "Metadata query for %s failed: %d\n", optarg, rc);
	return rc;

	} else if(rc == 2) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, NULL, provider, &metadata, 0);
	if(metadata) {
	printf("%s\n", metadata);
	return 0;
	}
	fprintf(stderr, "Metadata query for %s failed: %d\n", optarg, rc);
	return rc;

	} else if(rc < 2) {
	fprintf(stderr, "Please specify standard:type or standard:provider:type, not %s\n", optarg);
	return -1;
	}

	} else if(safe_str_eq("list-agents", longname)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;
	char standard[512];
	char provider[512];

	rc = sscanf(optarg, "%[^:]:%s", standard, provider);
	if(rc == 1) {
	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, NULL);
	provider[0] = '*';
	provider[1] = 0;

	} else if(rc == 2) {
	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, standard, provider);
	}

	if (rc > 0) {
	rc = 0;
	for (iter = list; iter != NULL; iter = iter->next) {
	printf("%s\n", iter->val);
	rc++;
	}
	lrmd_list_freeall(list);
	fprintf(stderr, "%d agents found for standard=%s, provider=%s\n", rc, standard, provider);
	rc = 0;
	} else {
	fprintf(stderr, "No agents found for standard=%s, provider=%s\n", standard, provider);
	rc = -1;
	}
	return rc;

	} else {
	crm_err("Unhandled long option: %s", longname);
	}
	break;
	case 'V':
	crm_bump_log_level();
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'x':
	xml_file = strdup(optarg);
	break;
	case 'Q':
	BE_QUIET = TRUE;
	break;
	case 'm':
	attr_set_type = XML_TAG_META_SETS;
	break;
	case 'z':
	attr_set_type = XML_TAG_UTILIZATION;
	break;
	case 'u':
	move_lifetime = strdup(optarg);
	break;
	case 'f':
	do_force = TRUE;
	break;
	case 'i':
	prop_id = optarg;
	break;
	case 's':
	prop_set = optarg;
	break;
	case 'r':
	rsc_id = optarg;
	break;
	case 'v':
	prop_value = optarg;
	break;
	case 't':
	rsc_type = optarg;
	break;
	case 'R':
	case 'P':
	rsc_cmd = flag;
	break;
	case 'L':
	case 'c':
	case 'l':
	case 'q':
	case 'w':
	case 'D':
	case 'F':
	case 'C':
	case 'W':
	case 'M':
	case 'U':
	case 'O':
	case 'o':
	case 'A':
	case 'a':
	rsc_cmd = flag;
	break;
	case 'p':
	case 'g':
	case 'd':
	case 'S':
	case 'G':
	prop_name = optarg;
	rsc_cmd = flag;
	break;
	case 'h':
	case 'H':
	case 'N':
	crm_trace("Option %c => %s", flag, optarg);
	host_uname = optarg;
	break;

	default:
	CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc && argv[optind] != NULL) {
	CMD_ERR("non-option ARGV-elements: ");
	while (optind < argc && argv[optind] != NULL) {
	CMD_ERR("%s ", argv[optind++]);
	++argerr;
	}
	CMD_ERR("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	our_pid = calloc(1, 11);
	if (our_pid != NULL) {
	snprintf(our_pid, 10, "%d", getpid());
	our_pid[10] = '\0';
	}

	if (do_force) {
	crm_debug("Forcing...");
	cib_options \|= cib_quorum_override;
	}

	set_working_set_defaults(&data_set);
	if (rsc_cmd != 'P') {
	resource_t *rsc = NULL;

	cib_conn = cib_new();
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	CMD_ERR("Error signing on to the CIB service: %s\n", pcmk_strerror(rc));
	return rc;
	}

	if (xml_file != NULL) {
	cib_xml_copy = filename2xml(xml_file);

	} else {
	cib_xml_copy = get_cib_copy(cib_conn);
	}

	if (cli_config_update(&cib_xml_copy, NULL, FALSE) == FALSE) {
	rc = -ENOKEY;
	goto bail;
	}

	data_set.input = cib_xml_copy;
	data_set.now = new_ha_date(TRUE);

	cluster_status(&data_set);
	if (rsc_id) {
	rsc = find_rsc_or_clone(rsc_id, &data_set);
	}
	if (rsc == NULL) {
	rc = -ENXIO;
	}
	}

	if (rsc_cmd == 'R' \|\| rsc_cmd == 'C' \|\| rsc_cmd == 'F' \|\| rsc_cmd == 'P') {
	xmlNode *xml = NULL;
	- mainloop_io_t *source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, 0, NULL, &crm_callbacks);
	+ mainloop_io_t *source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
	crmd_channel = mainloop_get_ipc_client(source);

	if (crmd_channel == NULL) {
	CMD_ERR("Error signing on to the CRMd service\n");
	rc = -ENOTCONN;
	goto bail;
	}

	xml = create_hello_message(our_pid, crm_system_name, "0", "1");
	crm_ipc_send(crmd_channel, xml, NULL, 0);
	free_xml(xml);
	}

	if (rsc_cmd == 'L') {
	rc = pcmk_ok;
	do_find_resource_list(&data_set, FALSE);

	} else if (rsc_cmd == 'l') {
	int found = 0;
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	found++;
	print_raw_rsc(rsc);
	}

	if (found == 0) {
	printf("NO resources configured\n");
	rc = -ENXIO;
	goto bail;
	}

	} else if (rsc_cmd == 0 && rsc_long_cmd) {
	svc_action_t *op = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	const char *action = NULL;
	GHashTable *params = NULL;
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	if (rsc == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}

	if(safe_str_eq(rsc_long_cmd, "force-stop")) {
	action = "stop";
	} else if(safe_str_eq(rsc_long_cmd, "force-start")) {
	action = "start";
	} else if(safe_str_eq(rsc_long_cmd, "force-check")) {
	action = "monitor";
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	params = generate_resource_params(rsc, &data_set);

	op = resources_action_create(
	rsc->id, rclass, rprov, rtype, action, 0, -1, params);

	if(services_action_sync(op)) {
	int more, lpc, last;
	char *local_copy = NULL;
	if(op->status == PCMK_LRM_OP_DONE) {
	printf("Operation %s for %s (%s:%s:%s) returned %d\n",
	action, rsc->id, rclass, rprov?rprov:"", rtype, op->rc);
	} else {
	printf("Operation %s for %s (%s:%s:%s) failed: %d\n",
	action, rsc->id, rclass, rprov?rprov:"", rtype, op->status);
	}

	if (op->stdout_data) {
	local_copy = strdup(op->stdout_data);
	more = strlen(local_copy);
	last = 0;

	for(lpc = 0; lpc < more; lpc++) {
	if(local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stdout: %s\n", local_copy+last);
	last = lpc+1;
	}
	}
	free(local_copy);
	}
	if (op->stderr_data) {
	local_copy = strdup(op->stderr_data);
	more = strlen(local_copy);
	last = 0;

	for(lpc = 0; lpc < more; lpc++) {
	if(local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stderr: %s\n", local_copy+last);
	last = lpc+1;
	}
	}
	free(local_copy);
	}
	}
	rc = op->rc;
	services_action_free(op);
	return rc;

	} else if (rsc_cmd == 'A' \|\| rsc_cmd == 'a') {
	GListPtr lpc = NULL;
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);

	if (rsc == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}

	unpack_constraints(cib_constraints, &data_set);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	show_colocation(rsc, TRUE, rsc_cmd == 'A', 1);

	fprintf(stdout, "* %s\n", rsc->id);
	show_location(rsc, NULL);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	show_colocation(rsc, FALSE, rsc_cmd == 'A', 1);

	} else if (rsc_cmd == 'c') {
	int found = 0;
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	print_cts_rsc(rsc);
	found++;
	}
	print_cts_constraints(&data_set);

	} else if (rsc_cmd == 'C') {
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	rc = delete_lrm_rsc(crmd_channel, host_uname, rsc, &data_set);
	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'F') {
	rc = fail_lrm_rsc(crmd_channel, host_uname, rsc_id, &data_set);
	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'O') {
	rc = list_resource_operations(rsc_id, host_uname, TRUE, &data_set);

	} else if (rsc_cmd == 'o') {
	rc = list_resource_operations(rsc_id, host_uname, FALSE, &data_set);

	} else if (rc == -ENXIO) {
	CMD_ERR("Resource %s not found: %s\n", crm_str(rsc_id), pcmk_strerror(rc));

	} else if (rsc_cmd == 'W') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = do_find_resource(rsc_id, NULL, &data_set);

	} else if (rsc_cmd == 'q') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource(rsc_id, &data_set, TRUE);

	} else if (rsc_cmd == 'w') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource(rsc_id, &data_set, FALSE);

	} else if (rsc_cmd == 'U') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	/* coverity[var_deref_model] False positive */
	rc = move_resource(rsc_id, NULL, NULL, cib_conn);

	} else if (rsc_cmd == 'M') {
	node_t *dest = NULL;
	node_t *current = NULL;
	const char *current_uname = NULL;
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	if (rsc != NULL && rsc->running_on != NULL) {
	current = rsc->running_on->data;
	if (current != NULL) {
	current_uname = current->details->uname;
	}
	}

	if (host_uname != NULL) {
	dest = pe_find_node(data_set.nodes, host_uname);
	}

	if (rsc == NULL) {
	CMD_ERR("Resource %s not moved:" " not found\n", rsc_id);

	} else if (rsc->variant == pe_native && g_list_length(rsc->running_on) > 1) {
	CMD_ERR("Resource %s not moved:" " active on multiple nodes\n", rsc_id);

	} else if (host_uname != NULL && dest == NULL) {
	CMD_ERR("Error performing operation: " "%s is not a known node\n", host_uname);
	rc = -ENXIO;

	} else if (host_uname != NULL && safe_str_eq(current_uname, host_uname)) {
	CMD_ERR("Error performing operation: "
	"%s is already active on %s\n", rsc_id, host_uname);

	} else if (current_uname != NULL && (do_force \|\| host_uname == NULL)) {
	/* coverity[var_deref_model] False positive */
	rc = move_resource(rsc_id, current_uname, host_uname, cib_conn);

	} else if (host_uname != NULL) {
	/* coverity[var_deref_model] False positive */
	rc = move_resource(rsc_id, NULL, host_uname, cib_conn);

	} else {
	CMD_ERR("Resource %s not moved: "
	"not-active and no preferred location" " specified.\n", rsc_id);
	rc = -EINVAL;
	}

	} else if (rsc_cmd == 'G') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource_prop(rsc_id, prop_name, &data_set);

	} else if (rsc_cmd == 'S') {
	xmlNode *msg_data = NULL;

	if (prop_value == NULL \|\| strlen(prop_value) == 0) {
	CMD_ERR("You need to supply a value with the -v option\n");
	rc = -EINVAL;
	goto bail;

	} else if (cib_conn == NULL) {
	rc = -ENOTCONN;
	goto bail;
	}

	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	CRM_LOG_ASSERT(rsc_type != NULL);
	CRM_LOG_ASSERT(prop_name != NULL);
	CRM_LOG_ASSERT(prop_value != NULL);

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
	crm_xml_add(msg_data, prop_name, prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else if (rsc_cmd == 'g') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource_attr(rsc_id, prop_name, &data_set);

	} else if (rsc_cmd == 'p') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	if (prop_value == NULL \|\| strlen(prop_value) == 0) {
	CMD_ERR("You need to supply a value with the -v option\n");
	rc = -EINVAL;
	goto bail;
	}
	/* coverity[var_deref_model] False positive */
	rc = set_resource_attr(rsc_id, prop_set, prop_id, prop_name,
	prop_value, cib_conn, &data_set);

	} else if (rsc_cmd == 'd') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	/* coverity[var_deref_model] False positive */
	rc = delete_resource_attr(rsc_id, prop_set, prop_id, prop_name, cib_conn, &data_set);

	} else if (rsc_cmd == 'P') {
	xmlNode *cmd = create_request(CRM_OP_REPROBE, NULL, host_uname,
	CRM_SYSTEM_CRMD, crm_system_name, our_pid);

	if (crm_ipc_send(crmd_channel, cmd, NULL, 0) > 0) {
	start_mainloop();
	}

	free_xml(cmd);

	} else if (rsc_cmd == 'R') {
	rc = refresh_lrm(crmd_channel, host_uname);
	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'D') {
	xmlNode *msg_data = NULL;

	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;

	}
	if (rsc_type == NULL) {
	CMD_ERR("You need to specify a resource type with -t");
	rc = -ENXIO;
	goto bail;

	} else if (cib_conn == NULL) {
	rc = -ENOTCONN;
	goto bail;
	}

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);

	rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else {
	CMD_ERR("Unknown command: %c\n", rsc_cmd);
	}

	bail:

	if (cib_conn != NULL) {
	cleanup_alloc_calculations(&data_set);
	cib_conn->cmds->signoff(cib_conn);
	cib_delete(cib_conn);
	}

	crm_xml_cleanup();

	if (rc == -pcmk_err_no_quorum) {
	CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
	CMD_ERR("Try using -f\n");

	} else if (rc != pcmk_ok) {
	CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
	}

	return rc;
	}
	diff --git a/tools/crmadmin.c b/tools/crmadmin.c
	index 4d8334d8ee..e9e577438f 100644
	--- a/tools/crmadmin.c
	+++ b/tools/crmadmin.c
	@@ -1,614 +1,614 @@

	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib.h>

	int message_timer_id = -1;
	int message_timeout_ms = 30 * 1000;

	GMainLoop *mainloop = NULL;
	crm_ipc_t *crmd_channel = NULL;
	char *admin_uuid = NULL;

	void usage(const char *cmd, int exit_status);
	gboolean do_init(void);
	int do_work(void);
	void crmadmin_ipc_connection_destroy(gpointer user_data);

	int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata);
	char pluralSection(const char a_section);
	xmlNode *handleCibMod(void);
	int do_find_node_list(xmlNode * xml_node);
	gboolean admin_message_timeout(gpointer data);
	gboolean is_node_online(xmlNode * node_state);

	enum debug {
	debug_none,
	debug_dec,
	debug_inc
	};

	gboolean BE_VERBOSE = FALSE;
	int expected_responses = 1;

	gboolean BASH_EXPORT = FALSE;
	gboolean DO_HEALTH = FALSE;
	gboolean DO_RESET = FALSE;
	gboolean DO_RESOURCE = FALSE;
	gboolean DO_ELECT_DC = FALSE;
	gboolean DO_WHOIS_DC = FALSE;
	gboolean DO_NODE_LIST = FALSE;
	gboolean BE_SILENT = FALSE;
	gboolean DO_RESOURCE_LIST = FALSE;
	enum debug DO_DEBUG = debug_none;
	const char *crmd_operation = NULL;

	xmlNode *msg_options = NULL;

	const char *standby_on_off = "on";
	const char *admin_verbose = XML_BOOLEAN_FALSE;
	char *id = NULL;
	char *disconnect = NULL;
	char *dest_node = NULL;
	char *rsc_name = NULL;
	char *crm_option = NULL;

	int operation_status = 0;
	const char *sys_to = NULL;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"quiet", 0, 0, 'q', "\tDisplay only the essential query information"},
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},

	{"-spacer-", 1, 0, '-', "\nCommands:"},
	/* daemon options */
	{"debug_inc", 1, 0, 'i', "Increase the crmd's debug level on the specified host"},
	{"debug_dec", 1, 0, 'd', "Decrease the crmd's debug level on the specified host"},
	{"status", 1, 0, 'S', "Display the status of the specified node." },
	{"-spacer-", 1, 0, '-', "\n\tResult is the node's internal FSM state which can be useful for debugging\n"},
	{"dc_lookup", 0, 0, 'D', "Display the uname of the node co-ordinating the cluster."},
	{"-spacer-", 1, 0, '-', "\n\tThis is an internal detail and is rarely useful to administrators except when deciding on which node to examine the logs.\n"},
	{"nodes", 0, 0, 'N', "\tDisplay the uname of all member nodes"},
	{"election", 0, 0, 'E', "(Advanced) Start an election for the cluster co-ordinator"},
	{"kill", 1, 0, 'K', "(Advanced) Shut down the crmd (not the rest of the clusterstack ) on the specified node"},
	{"health", 0, 0, 'H', NULL, 1},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{XML_ATTR_TIMEOUT, 1, 0, 't', "Time (in milliseconds) to wait before declaring the operation failed"},
	{"bash-export", 0, 0, 'B', "Create Bash export entries of the form 'export uname=uuid'\n"},

	{"-spacer-", 1, 0, '-', "Notes:"},
	{"-spacer-", 1, 0, '-', " The -i,-d,-K and -E commands are rarely used and may be removed in future versions."},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	int option_index = 0;
	int argerr = 0;
	int flag;

	crm_log_cli_init("crmadmin");
	crm_set_options(NULL, "command [options]", long_options,
	"Development tool for performing some crmd-specific commands."
	"\n Likely to be replaced by crm_node in the future");
	if (argc < 2) {
	crm_help('?', EX_USAGE);
	}

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	BE_VERBOSE = TRUE;
	admin_verbose = XML_BOOLEAN_TRUE;
	crm_bump_log_level();
	break;
	case 't':
	message_timeout_ms = atoi(optarg);
	if (message_timeout_ms < 1) {
	message_timeout_ms = 30 * 1000;
	}
	break;

	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'D':
	DO_WHOIS_DC = TRUE;
	break;
	case 'B':
	BASH_EXPORT = TRUE;
	break;
	case 'K':
	DO_RESET = TRUE;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	crmd_operation = CRM_OP_LOCAL_SHUTDOWN;
	break;
	case 'q':
	BE_SILENT = TRUE;
	break;
	case 'i':
	DO_DEBUG = debug_inc;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'd':
	DO_DEBUG = debug_dec;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'S':
	DO_HEALTH = TRUE;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'E':
	DO_ELECT_DC = TRUE;
	break;
	case 'N':
	DO_NODE_LIST = TRUE;
	break;
	case 'H':
	DO_HEALTH = TRUE;
	break;
	default:
	printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	if (do_init()) {
	int res = 0;

	res = do_work();
	if (res > 0) {
	/* wait for the reply by creating a mainloop and running it until
	* the callbacks are invoked...
	*/
	mainloop = g_main_new(FALSE);
	crm_trace("Waiting for %d replies from the local CRM", expected_responses);

	message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);

	g_main_run(mainloop);

	} else if (res < 0) {
	crm_err("No message to send");
	operation_status = -1;
	}
	} else {
	crm_warn("Init failed, could not perform requested operations");
	operation_status = -2;
	}

	crm_trace("%s exiting normally", crm_system_name);
	return operation_status;
	}

	int
	do_work(void)
	{
	int ret = 1;

	/* construct the request */
	xmlNode *msg_data = NULL;
	gboolean all_is_good = TRUE;

	msg_options = create_xml_node(NULL, XML_TAG_OPTIONS);
	crm_xml_add(msg_options, XML_ATTR_VERBOSE, admin_verbose);
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	if (DO_HEALTH == TRUE) {
	crm_trace("Querying the system");

	sys_to = CRM_SYSTEM_DC;

	if (dest_node != NULL) {
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_PING;

	if (BE_VERBOSE) {
	expected_responses = 1;
	}

	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	} else {
	crm_info("Cluster-wide health not available yet");
	all_is_good = FALSE;
	}

	} else if (DO_ELECT_DC) {
	/* tell the local node to initiate an election */

	dest_node = NULL;
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_VOTE;

	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
	ret = 0; /* no return message */

	} else if (DO_WHOIS_DC) {
	dest_node = NULL;
	sys_to = CRM_SYSTEM_DC;
	crmd_operation = CRM_OP_PING;
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");


	} else if (DO_NODE_LIST) {

	cib_t *the_cib = cib_new();
	xmlNode *output = NULL;

	int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);

	if (rc != pcmk_ok) {
	return -1;
	}

	output = get_cib_copy(the_cib);
	do_find_node_list(output);

	free_xml(output);
	the_cib->cmds->signoff(the_cib);
	exit(rc);

	} else if (DO_RESET) {
	/* tell dest_node to initiate the shutdown proceedure
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	ret = 0; /* no return message */

	} else if (DO_DEBUG == debug_inc) {
	/* tell dest_node to increase its debug level
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_DEBUG_UP;

	ret = 0; /* no return message */

	} else if (DO_DEBUG == debug_dec) {
	/* tell dest_node to increase its debug level
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_DEBUG_DOWN;

	ret = 0; /* no return message */

	} else {
	crm_err("Unknown options");
	all_is_good = FALSE;
	}

	if (all_is_good == FALSE) {
	crm_err("Creation of request failed. No message to send");
	return -1;
	}

	/* send it */
	if (crmd_channel == NULL) {
	crm_err("The IPC connection is not valid, cannot send anything");
	return -1;
	}

	if (sys_to == NULL) {
	if (dest_node != NULL) {
	sys_to = CRM_SYSTEM_CRMD;
	} else {
	sys_to = CRM_SYSTEM_DC;
	}
	}

	{
	xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to,
	crm_system_name, admin_uuid);

	crm_ipc_send(crmd_channel, cmd, NULL, 0);
	free_xml(cmd);
	}

	return ret;
	}

	void
	crmadmin_ipc_connection_destroy(gpointer user_data)
	{
	crm_err("Connection to CRMd was terminated");
	if (mainloop) {
	g_main_quit(mainloop);
	} else {
	exit(1);
	}
	}

	struct ipc_client_callbacks crm_callbacks =
	{
	.dispatch = admin_msg_callback,
	.destroy = crmadmin_ipc_connection_destroy
	};

	gboolean
	do_init(void)
	{
	- mainloop_io_t *source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, 0, NULL, &crm_callbacks);
	+ mainloop_io_t *source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);

	admin_uuid = calloc(1, 11);
	if (admin_uuid != NULL) {
	snprintf(admin_uuid, 10, "%d", getpid());
	admin_uuid[10] = '\0';
	}

	crmd_channel = mainloop_get_ipc_client(source);

	if (DO_RESOURCE \|\| DO_RESOURCE_LIST \|\| DO_NODE_LIST) {
	return TRUE;

	} else if (crmd_channel != NULL) {
	xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1");
	crm_ipc_send(crmd_channel, xml, NULL, 0);
	return TRUE;
	}
	return FALSE;
	}

	static xmlNode *
	validate_crm_message(xmlNode * msg, const char sys, const char uuid, const char *msg_type)
	{
	const char *to = NULL;
	const char *type = NULL;
	const char *crm_msg_reference = NULL;
	xmlNode *action = NULL;
	const char *true_sys;
	char *local_sys = NULL;

	if (msg == NULL) {
	return NULL;
	}

	to = crm_element_value(msg, F_CRM_SYS_TO);
	type = crm_element_value(msg, F_CRM_MSG_TYPE);

	crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);
	action = msg;
	true_sys = sys;

	if (uuid != NULL) {
	local_sys = generate_hash_key(sys, uuid);
	true_sys = local_sys;
	}

	if (to == NULL) {
	crm_info("No sub-system defined.");
	action = NULL;
	} else if (true_sys != NULL && strcasecmp(to, true_sys) != 0) {
	crm_trace("The message is not for this sub-system (%s != %s).", to, true_sys);
	action = NULL;
	}

	free(local_sys);

	if (type == NULL) {
	crm_info("No message type defined.");
	return NULL;

	} else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
	crm_info("Expecting a (%s) message but received a (%s).", msg_type, type);
	action = NULL;
	}

	if (crm_msg_reference == NULL) {
	crm_info("No message crm_msg_reference defined.");
	action = NULL;
	}
	/*
	if(action != NULL)
	crm_trace(
	"XML is valid and node with message type (%s) found.",
	type);
	crm_trace("Returning node (%s)", crm_element_name(action));
	*/

	return action;
	}

	int
	admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata)
	{
	static int received_responses = 0;
	const char *result = NULL;
	xmlNode *xml = string2xml(buffer);

	received_responses++;
	g_source_remove(message_timer_id);

	crm_log_xml_trace(xml, "ipc");

	if (xml == NULL) {
	crm_info("XML in IPC message was not valid... " "discarding.");

	} else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) {
	crm_trace("Message was not a CRM response. Discarding.");

	} else {
	result = crm_element_value(xml, XML_ATTR_RESULT);
	if (result == NULL \|\| strcasecmp(result, "ok") == 0) {
	result = "pass";

	} else {
	result = "fail";
	}

	if (DO_HEALTH) {
	xmlNode *data = get_message_xml(xml, F_CRM_DATA);
	const char *state = crm_element_value(data, "crmd_state");

	printf("Status of %s@%s: %s (%s)\n",
	crm_element_value(data, XML_PING_ATTR_SYSFROM),
	crm_element_value(xml, F_CRM_HOST_FROM),
	state, crm_element_value(data, XML_PING_ATTR_STATUS));

	if (BE_SILENT && state != NULL) {
	fprintf(stderr, "%s\n", state);
	}

	} else if (DO_WHOIS_DC) {
	const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);

	printf("Designated Controller is: %s\n", dc);
	if (BE_SILENT && dc != NULL) {
	fprintf(stderr, "%s\n", dc);
	}
	exit(0);
	}
	}

	free_xml(xml);

	if (received_responses >= expected_responses) {
	crm_trace("Received expected number (%d) of messages from Heartbeat."
	" Exiting normally.", expected_responses);
	exit(0);
	}

	message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
	return 0;
	}

	gboolean
	admin_message_timeout(gpointer data)
	{
	fprintf(stderr, "No messages received in %d seconds.. aborting\n",
	(int)message_timeout_ms / 1000);
	crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
	operation_status = -3;
	g_main_quit(mainloop);
	return FALSE;
	}

	gboolean
	is_node_online(xmlNode * node_state)
	{
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
	const char *join_state = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *crm_state = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *ccm_state = crm_element_value(node_state, XML_NODE_IN_CLUSTER);

	if (safe_str_neq(join_state, CRMD_JOINSTATE_DOWN)
	&& crm_is_true(ccm_state)
	&& safe_str_eq(crm_state, "online")) {
	crm_trace("Node %s is online", uname);
	return TRUE;
	}
	crm_trace("Node %s: ccm=%s join=%s exp=%s crm=%s",
	uname, crm_str(ccm_state),
	crm_str(join_state), crm_str(exp_state), crm_str(crm_state));
	crm_trace("Node %s is offline", uname);
	return FALSE;
	}

	int
	do_find_node_list(xmlNode * xml_node)
	{
	int found = 0;
	xmlNode *node = NULL;
	xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node);

	for (node = __xml_first_child(nodes); node != NULL; node = __xml_next(node)) {
	if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) {

	if (BASH_EXPORT) {
	printf("export %s=%s\n",
	crm_element_value(node, XML_ATTR_UNAME),
	crm_element_value(node, XML_ATTR_ID));
	} else {
	printf("%s node: %s (%s)\n",
	crm_element_value(node, XML_ATTR_TYPE),
	crm_element_value(node, XML_ATTR_UNAME),
	crm_element_value(node, XML_ATTR_ID));
	}
	found++;
	}
	}

	if (found == 0) {
	printf("NO nodes configured\n");
	}

	return found;
	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 6:10 PM (1 d, 4 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665114
Default Alt Text: (425 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions